## Summary
Overhauls the backup/archive system to address stability, robustness, fault tolerance, performance, and pluggability.
### Problems solved
- **Critical bug**: `Interlocked.CompareExchange` arguments were reversed — the concurrent archive guard never actually prevented concurrent archives
- **Brittle compression**: Relied on spawning external `zstd.exe`/`tar.exe`/`bsdtar.exe` processes with platform-specific workarounds (Windows two-step hack, auto-downloading bsdtar from libarchive.org)
- **No fault tolerance**: Partial archives possible if process killed mid-write; no recovery mechanism; originals deleted before verification
- **No extensibility**: No way to add remote backup destinations (S3, rsync) without modifying core code
- **Silent failures**: Bare `catch` blocks swallowed exceptions with no diagnostics
### What changed
**Cross-platform streaming compression** — Replaced external process spawning with `System.Formats.Tar` (built-in .NET) + `ZstdNet` (native libzstd P/Invoke). Single-pass streaming: `TarWriter → CompressionStream → FileStream`. No intermediate `.tar` file, no platform-specific code paths.
**Archive journal** — JSON-based operation journal (`Archives/.archive-journal.json`) tracks state machine: `Started → Archived → Distributed → Completed` (or `Failed`). On startup, recovers interrupted operations (cleans temp files, completes pruning).
**Verify before delete** — Archives are verified (entry count check) before deleting source backups. Temp-file-then-atomic-rename pattern prevents partial archives.
**Retry logic** — File operations (moves, deletes) retry with linear backoff for transient I/O failures (antivirus locks, file copy operations).
**Plugin architecture** — `IArchiveDestination` interface in `Projects/Server/` enables external plugins (loaded via `assemblies.json`) to receive completed archives. `ArchiveDestinationRegistry` tracks destinations. Conservative pruning: source backups preserved if any destination with retention fails.
**Configurable** — Retention counts (hourly/daily/monthly), compression level, retry settings, backup max age all configurable via `ServerConfiguration`.
### New admin commands
- `[ArchiveStatus` — Shows journal state, destinations, next scheduled archive times
- `[ArchiveNow` — Forces immediate rollup regardless of schedule
### Files
| Change | File |
|--------|------|
| New | `Server/Saves/ArchiveEnums.cs`, `IArchiveDestination.cs`, `ArchiveDestinationRegistry.cs`, `ArchiveEventArgs.cs` |
| New | `Server/Events/ArchiveEvents.cs` (partial EventSink) |
| New | `UOContent/Compression/ManagedArchive.cs` |
| New | `UOContent/World Saves/ArchiveJournal.cs`, `LocalArchiveDestination.cs` |
| Rewrite | `UOContent/World Saves/AutoArchive.cs` |
| Modified | `UOContent/World Saves/SaveCommands.cs`, `Server/Utilities/PathUtility.cs`, `UOContent.csproj` |
| Deleted | `UOContent/Compression/TarArchive.cs`, `ZstdArchive.cs` |
| Tests | 29 new tests (ManagedArchive, ArchiveJournal, AutoArchiveHelpers) |
### Backward compatibility
- Existing `.tar.zst` archives are fully readable by the new managed code
- Config keys preserved; new keys use sensible defaults
- `autoArchive.compressionFormat` setting removed (Zstd only)
- Legacy `bsdtar/` directory logged as removable on startup
### Plugin example (external repo)
```csharp
public static class S3Plugin
{
public static void Configure()
{
ArchiveDestinationRegistry.Register(new S3Destination("my-bucket", "us-east-1"));
}
}
```
## Test plan
- [x] 29 new unit tests covering archive create/extract/count, journal state machine, recovery, destinations
- [x] All 969 tests pass (671 Server + 298 UOContent)
- [x] Manual: run server, trigger save, verify backup created and archive rolled up
- [x] Manual: kill server mid-archive, restart, verify journal recovery
- [x] Manual: test with large save files (~500MB+) to benchmark streaming vs old approach
269 lines
9.3 KiB
C#
269 lines
9.3 KiB
C#
/*************************************************************************
|
|
* ModernUO *
|
|
* Copyright 2019-2026 - ModernUO Development Team *
|
|
* Email: hi@modernuo.com *
|
|
* File: ArchiveJournal.cs *
|
|
* *
|
|
* This program is free software: you can redistribute it and/or modify *
|
|
* it under the terms of the GNU General Public License as published by *
|
|
* the Free Software Foundation, either version 3 of the License, or *
|
|
* (at your option) any later version. *
|
|
* *
|
|
* You should have received a copy of the GNU General Public License *
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
|
*************************************************************************/
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using Server.Json;
|
|
using Server.Logging;
|
|
|
|
namespace Server.Saves;
|
|
|
|
public enum ArchiveOperationState
|
|
{
|
|
Started,
|
|
Archived,
|
|
Distributed,
|
|
Completed,
|
|
Failed
|
|
}
|
|
|
|
public class ArchiveJournalEntry
|
|
{
|
|
public string Id { get; set; }
|
|
public ArchivePeriod Period { get; set; }
|
|
public DateTime RangeStart { get; set; }
|
|
public ArchiveOperationState State { get; set; }
|
|
public string ArchiveFile { get; set; }
|
|
public string TempFile { get; set; }
|
|
public List<string> SourceDirectories { get; set; } = [];
|
|
public int EntryCount { get; set; }
|
|
public long TotalSourceBytes { get; set; }
|
|
public long ArchiveBytes { get; set; }
|
|
public DateTime StartedAt { get; set; }
|
|
public DateTime CompletedAt { get; set; } = DateTime.MaxValue;
|
|
public string FailureReason { get; set; }
|
|
public Dictionary<string, bool> DestinationResults { get; set; } = new();
|
|
}
|
|
|
|
public class ArchiveJournalData
|
|
{
|
|
public List<ArchiveJournalEntry> Operations { get; set; } = [];
|
|
}
|
|
|
|
public static class ArchiveJournal
|
|
{
|
|
private const int MaxJournalEntries = 100;
|
|
|
|
private static readonly ILogger logger = LogFactory.GetLogger(typeof(ArchiveJournal));
|
|
|
|
private static string _journalPath;
|
|
private static ArchiveJournalData _data = new();
|
|
|
|
public static IReadOnlyList<ArchiveJournalEntry> Operations => _data.Operations;
|
|
|
|
public static void Configure(string archivePath)
|
|
{
|
|
_journalPath = Path.Combine(archivePath, ".archive-journal.json");
|
|
_data = new ArchiveJournalData();
|
|
Load();
|
|
}
|
|
|
|
private static void Load()
|
|
{
|
|
try
|
|
{
|
|
if (File.Exists(_journalPath))
|
|
{
|
|
_data = JsonConfig.Deserialize<ArchiveJournalData>(_journalPath) ?? new ArchiveJournalData();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.Error(ex, "Failed to load archive journal, starting fresh");
|
|
_data = new ArchiveJournalData();
|
|
}
|
|
}
|
|
|
|
private static void Save()
|
|
{
|
|
try
|
|
{
|
|
var tempPath = _journalPath + ".tmp";
|
|
JsonConfig.Serialize(tempPath, _data);
|
|
|
|
if (File.Exists(_journalPath))
|
|
{
|
|
File.Delete(_journalPath);
|
|
}
|
|
|
|
File.Move(tempPath, _journalPath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.Error(ex, "Failed to save archive journal");
|
|
}
|
|
}
|
|
|
|
private static void Prune()
|
|
{
|
|
if (_data.Operations.Count > MaxJournalEntries)
|
|
{
|
|
// Remove oldest completed/failed entries beyond the limit
|
|
var toRemove = _data.Operations.Count - MaxJournalEntries;
|
|
for (var i = _data.Operations.Count - 1; i >= 0 && toRemove > 0; i--)
|
|
{
|
|
if (_data.Operations[i].State is ArchiveOperationState.Completed or ArchiveOperationState.Failed)
|
|
{
|
|
_data.Operations.RemoveAt(i);
|
|
toRemove--;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public static ArchiveJournalEntry BeginOperation(
|
|
ArchivePeriod period,
|
|
DateTime rangeStart,
|
|
string tempFile,
|
|
string archiveFile,
|
|
List<string> sourceDirectories)
|
|
{
|
|
var entry = new ArchiveJournalEntry
|
|
{
|
|
Id = $"{rangeStart:O}-{period.ToString().ToLowerInvariant()}",
|
|
Period = period,
|
|
RangeStart = rangeStart,
|
|
State = ArchiveOperationState.Started,
|
|
TempFile = tempFile,
|
|
ArchiveFile = archiveFile,
|
|
SourceDirectories = sourceDirectories,
|
|
StartedAt = Core.Now
|
|
};
|
|
|
|
_data.Operations.Add(entry);
|
|
Prune();
|
|
Save();
|
|
return entry;
|
|
}
|
|
|
|
public static void RecordArchived(ArchiveJournalEntry entry, int entryCount, long archiveBytes)
|
|
{
|
|
entry.State = ArchiveOperationState.Archived;
|
|
entry.EntryCount = entryCount;
|
|
entry.ArchiveBytes = archiveBytes;
|
|
entry.TempFile = null; // Temp file has been renamed to final
|
|
Save();
|
|
}
|
|
|
|
public static void RecordDistributed(ArchiveJournalEntry entry, Dictionary<string, bool> results)
|
|
{
|
|
entry.State = ArchiveOperationState.Distributed;
|
|
entry.DestinationResults = results;
|
|
Save();
|
|
}
|
|
|
|
public static void RecordCompleted(ArchiveJournalEntry entry)
|
|
{
|
|
entry.State = ArchiveOperationState.Completed;
|
|
entry.CompletedAt = Core.Now;
|
|
Save();
|
|
}
|
|
|
|
public static void RecordFailure(ArchiveJournalEntry entry, string reason)
|
|
{
|
|
entry.State = ArchiveOperationState.Failed;
|
|
entry.FailureReason = reason;
|
|
entry.CompletedAt = Core.Now;
|
|
Save();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Recovers from interrupted operations on startup.
|
|
/// Handles missing files/directories gracefully (e.g. admin cleaned up archives folder).
|
|
/// </summary>
|
|
public static void RecoverInterrupted()
|
|
{
|
|
var recovered = false;
|
|
|
|
foreach (var entry in _data.Operations)
|
|
{
|
|
switch (entry.State)
|
|
{
|
|
case ArchiveOperationState.Started:
|
|
{
|
|
// Archive creation was interrupted. Clean up temp file if it exists.
|
|
if (entry.TempFile != null)
|
|
{
|
|
try
|
|
{
|
|
if (File.Exists(entry.TempFile))
|
|
{
|
|
File.Delete(entry.TempFile);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.Warning(ex, "Failed to clean up temp file {File}", entry.TempFile);
|
|
}
|
|
}
|
|
|
|
logger.Warning(
|
|
"Recovered interrupted archive operation {Id} (was in Started state)",
|
|
entry.Id
|
|
);
|
|
entry.State = ArchiveOperationState.Failed;
|
|
entry.FailureReason = "Interrupted during archive creation";
|
|
entry.CompletedAt = Core.Now;
|
|
recovered = true;
|
|
break;
|
|
}
|
|
case ArchiveOperationState.Archived:
|
|
{
|
|
// Archive was created but not distributed.
|
|
// If the archive file was deleted (admin cleanup), mark as failed.
|
|
if (entry.ArchiveFile != null && !File.Exists(entry.ArchiveFile))
|
|
{
|
|
logger.Warning(
|
|
"Archive file for operation {Id} no longer exists (manually deleted?). Marking failed.",
|
|
entry.Id
|
|
);
|
|
entry.State = ArchiveOperationState.Failed;
|
|
entry.FailureReason = "Archive file missing on recovery";
|
|
entry.CompletedAt = Core.Now;
|
|
recovered = true;
|
|
}
|
|
else
|
|
{
|
|
logger.Warning(
|
|
"Found unfinished archive operation {Id} (Archived but not distributed). " +
|
|
"Will re-attempt distribution on next cycle.",
|
|
entry.Id
|
|
);
|
|
}
|
|
|
|
break;
|
|
}
|
|
case ArchiveOperationState.Distributed:
|
|
{
|
|
// Sources should have been pruned but weren't.
|
|
logger.Information(
|
|
"Completing interrupted operation {Id} (was Distributed, pruning sources)",
|
|
entry.Id
|
|
);
|
|
entry.State = ArchiveOperationState.Completed;
|
|
entry.CompletedAt = Core.Now;
|
|
recovered = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (recovered)
|
|
{
|
|
Save();
|
|
}
|
|
}
|
|
}
|