## Summary
Overhauls the backup/archive system to address stability, robustness, fault tolerance, performance, and pluggability.
### Problems solved
- **Critical bug**: `Interlocked.CompareExchange` arguments were reversed — the concurrent archive guard never actually prevented concurrent archives
- **Brittle compression**: Relied on spawning external `zstd.exe`/`tar.exe`/`bsdtar.exe` processes with platform-specific workarounds (Windows two-step hack, auto-downloading bsdtar from libarchive.org)
- **No fault tolerance**: Partial archives possible if process killed mid-write; no recovery mechanism; originals deleted before verification
- **No extensibility**: No way to add remote backup destinations (S3, rsync) without modifying core code
- **Silent failures**: Bare `catch` blocks swallowed exceptions with no diagnostics
### What changed
**Cross-platform streaming compression** — Replaced external process spawning with `System.Formats.Tar` (built-in .NET) + `ZstdNet` (native libzstd P/Invoke). Single-pass streaming: `TarWriter → CompressionStream → FileStream`. No intermediate `.tar` file, no platform-specific code paths.
**Archive journal** — JSON-based operation journal (`Archives/.archive-journal.json`) tracks state machine: `Started → Archived → Distributed → Completed` (or `Failed`). On startup, recovers interrupted operations (cleans temp files, completes pruning).
**Verify before delete** — Archives are verified (entry count check) before deleting source backups. Temp-file-then-atomic-rename pattern prevents partial archives.
**Retry logic** — File operations (moves, deletes) retry with linear backoff for transient I/O failures (antivirus locks, file copy operations).
**Plugin architecture** — `IArchiveDestination` interface in `Projects/Server/` enables external plugins (loaded via `assemblies.json`) to receive completed archives. `ArchiveDestinationRegistry` tracks destinations. Conservative pruning: source backups preserved if any destination with retention fails.
**Configurable** — Retention counts (hourly/daily/monthly), compression level, retry settings, backup max age all configurable via `ServerConfiguration`.
### New admin commands
- `[ArchiveStatus` — Shows journal state, destinations, next scheduled archive times
- `[ArchiveNow` — Forces immediate rollup regardless of schedule
### Files
| Change | File |
|--------|------|
| New | `Server/Saves/ArchiveEnums.cs`, `IArchiveDestination.cs`, `ArchiveDestinationRegistry.cs`, `ArchiveEventArgs.cs` |
| New | `Server/Events/ArchiveEvents.cs` (partial EventSink) |
| New | `UOContent/Compression/ManagedArchive.cs` |
| New | `UOContent/World Saves/ArchiveJournal.cs`, `LocalArchiveDestination.cs` |
| Rewrite | `UOContent/World Saves/AutoArchive.cs` |
| Modified | `UOContent/World Saves/SaveCommands.cs`, `Server/Utilities/PathUtility.cs`, `UOContent.csproj` |
| Deleted | `UOContent/Compression/TarArchive.cs`, `ZstdArchive.cs` |
| Tests | 29 new tests (ManagedArchive, ArchiveJournal, AutoArchiveHelpers) |
### Backward compatibility
- Existing `.tar.zst` archives are fully readable by the new managed code
- Config keys preserved; new keys use sensible defaults
- `autoArchive.compressionFormat` setting removed (Zstd only)
- Legacy `bsdtar/` directory logged as removable on startup
### Plugin example (external repo)
```csharp
public static class S3Plugin
{
public static void Configure()
{
ArchiveDestinationRegistry.Register(new S3Destination("my-bucket", "us-east-1"));
}
}
```
## Test plan
- [x] 29 new unit tests covering archive create/extract/count, journal state machine, recovery, destinations
- [x] All 969 tests pass (671 Server + 298 UOContent)
- [x] Manual: run server, trigger save, verify backup created and archive rolled up
- [x] Manual: kill server mid-archive, restart, verify journal recovery
- [x] Manual: test with large save files (~500MB+) to benchmark streaming vs old approach
104 lines
4.1 KiB
C#
104 lines
4.1 KiB
C#
using Server.Saves;
|
|
|
|
namespace Server.Commands;
|
|
|
|
public static class SaveCommands
|
|
{
|
|
public static void Configure()
|
|
{
|
|
CommandSystem.Register("Save", AccessLevel.Administrator, Save_OnCommand);
|
|
CommandSystem.Register("SetSaves", AccessLevel.Administrator, SetAutoSaves_OnCommand);
|
|
CommandSystem.Register("AutoSave", AccessLevel.Administrator, SetAutoSaves_OnCommand);
|
|
CommandSystem.Register("SaveFrequency", AccessLevel.Administrator, SetSaveFrequency_OnCommand);
|
|
CommandSystem.Register("PruneArchives", AccessLevel.Administrator, PruneArchives_OnCommand);
|
|
CommandSystem.Register("ArchiveStatus", AccessLevel.Administrator, ArchiveStatus_OnCommand);
|
|
CommandSystem.Register("ArchiveNow", AccessLevel.Administrator, ArchiveNow_OnCommand);
|
|
}
|
|
|
|
[Usage("PruneArchives"), Description("Prunes archives folder.")]
|
|
private static void PruneArchives_OnCommand(CommandEventArgs e) => AutoArchive.PruneBackups();
|
|
|
|
[Usage("Save"), Description("Saves the world.")]
|
|
private static void Save_OnCommand(CommandEventArgs e) => AutoSave.Save();
|
|
|
|
[Usage("AutoSave <on | off>"), Description("Enables or disables automatic shard saving.")]
|
|
public static void SetAutoSaves_OnCommand(CommandEventArgs e)
|
|
{
|
|
if (e.Length == 1)
|
|
{
|
|
var enabled = AutoSave.SavesEnabled = e.GetBoolean(0);
|
|
|
|
e.Mobile.SendMessage(enabled ? "Saves have been enabled." : "Saves have been disabled.");
|
|
}
|
|
else
|
|
{
|
|
e.Mobile.SendMessage("Format: AutoSave <on | off>");
|
|
}
|
|
}
|
|
|
|
[Usage("SaveFrequency <duration> [warning duration]"), Description("Sets the save frequency starting at midnight local to the server.")]
|
|
public static void SetSaveFrequency_OnCommand(CommandEventArgs e)
|
|
{
|
|
if (e.Length < 1)
|
|
{
|
|
e.Mobile.SendMessage("Format: SaveFrequency <duration> [warning duration]");
|
|
return;
|
|
}
|
|
|
|
var saveDelay = e.GetTimeSpan(0);
|
|
var warningDelay = e.Length >= 2 ? e.GetTimeSpan(1) : AutoSave.Warning;
|
|
|
|
AutoSave.ResetAutoSave(saveDelay, warningDelay);
|
|
|
|
e.Mobile.SendMessage("Save frequency has been updated.");
|
|
}
|
|
|
|
[Usage("ArchiveStatus")]
|
|
[Description("Shows archive system status, journal state, and registered destinations.")]
|
|
private static void ArchiveStatus_OnCommand(CommandEventArgs e)
|
|
{
|
|
var mobile = e.Mobile;
|
|
|
|
mobile.SendMessage("--- Archive Status ---");
|
|
mobile.SendMessage($"Next hourly: {AutoArchive.NextHourlyArchive:yyyy-MM-dd HH:mm}");
|
|
mobile.SendMessage($"Next daily: {AutoArchive.NextDailyArchive:yyyy-MM-dd}");
|
|
mobile.SendMessage($"Next monthly: {AutoArchive.NextMonthlyArchive:yyyy-MM}");
|
|
|
|
var destinations = ArchiveDestinationRegistry.Destinations;
|
|
mobile.SendMessage($"Registered destinations: {destinations.Count}");
|
|
foreach (var dest in destinations)
|
|
{
|
|
mobile.SendMessage(
|
|
$" - {dest.Name} (retention: {dest.GetRetentionCount(ArchivePeriod.Hourly)}h/" +
|
|
$"{dest.GetRetentionCount(ArchivePeriod.Daily)}d/" +
|
|
$"{dest.GetRetentionCount(ArchivePeriod.Monthly)}m)"
|
|
);
|
|
}
|
|
|
|
var operations = ArchiveJournal.Operations;
|
|
var pending = 0;
|
|
var failed = 0;
|
|
for (var i = 0; i < operations.Count; i++)
|
|
{
|
|
var op = operations[i];
|
|
if (op.State is ArchiveOperationState.Started or ArchiveOperationState.Archived
|
|
or ArchiveOperationState.Distributed)
|
|
{
|
|
pending++;
|
|
}
|
|
else if (op.State == ArchiveOperationState.Failed)
|
|
{
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
mobile.SendMessage($"Journal entries: {operations.Count} ({pending} pending, {failed} failed)");
|
|
}
|
|
|
|
[Usage("ArchiveNow"), Description("Forces an immediate archive rollup regardless of schedule.")]
|
|
private static void ArchiveNow_OnCommand(CommandEventArgs e)
|
|
{
|
|
e.Mobile.SendMessage("Forcing immediate archive rollup...");
|
|
AutoArchive.ForceRollup();
|
|
}
|
|
}
|