ModernUO/Projects/UOContent.Tests/Tests/WorldSaves/ArchiveJournalTests.cs
Kamron Batman 3b4e1137f6
feat: Implements new robust/pluggable backup/archive system. (#2388)
## Summary

Overhauls the backup/archive system to address stability, robustness, fault tolerance, performance, and pluggability.

### Problems solved
- **Critical bug**: `Interlocked.CompareExchange` arguments were reversed — the concurrent archive guard never actually prevented concurrent archives
- **Brittle compression**: Relied on spawning external `zstd.exe`/`tar.exe`/`bsdtar.exe` processes with platform-specific workarounds (Windows two-step hack, auto-downloading bsdtar from libarchive.org)
- **No fault tolerance**: Partial archives possible if process killed mid-write; no recovery mechanism; originals deleted before verification
- **No extensibility**: No way to add remote backup destinations (S3, rsync) without modifying core code
- **Silent failures**: Bare `catch` blocks swallowed exceptions with no diagnostics

### What changed

**Cross-platform streaming compression** — Replaced external process spawning with `System.Formats.Tar` (built-in .NET) + `ZstdNet` (native libzstd P/Invoke). Single-pass streaming: `TarWriter → CompressionStream → FileStream`. No intermediate `.tar` file, no platform-specific code paths.

**Archive journal** — JSON-based operation journal (`Archives/.archive-journal.json`) tracks state machine: `Started → Archived → Distributed → Completed` (or `Failed`). On startup, recovers interrupted operations (cleans temp files, completes pruning).

**Verify before delete** — Archives are verified (entry count check) before deleting source backups. Temp-file-then-atomic-rename pattern prevents partial archives.

**Retry logic** — File operations (moves, deletes) retry with linear backoff for transient I/O failures (antivirus locks, file copy operations).

**Plugin architecture** — `IArchiveDestination` interface in `Projects/Server/` enables external plugins (loaded via `assemblies.json`) to receive completed archives. `ArchiveDestinationRegistry` tracks destinations. Conservative pruning: source backups preserved if any destination with retention fails.

**Configurable** — Retention counts (hourly/daily/monthly), compression level, retry settings, backup max age all configurable via `ServerConfiguration`.

### New admin commands
- `[ArchiveStatus` — Shows journal state, destinations, next scheduled archive times
- `[ArchiveNow` — Forces immediate rollup regardless of schedule

### Files

| Change | File |
|--------|------|
| New | `Server/Saves/ArchiveEnums.cs`, `IArchiveDestination.cs`, `ArchiveDestinationRegistry.cs`, `ArchiveEventArgs.cs` |
| New | `Server/Events/ArchiveEvents.cs` (partial EventSink) |
| New | `UOContent/Compression/ManagedArchive.cs` |
| New | `UOContent/World Saves/ArchiveJournal.cs`, `LocalArchiveDestination.cs` |
| Rewrite | `UOContent/World Saves/AutoArchive.cs` |
| Modified | `UOContent/World Saves/SaveCommands.cs`, `Server/Utilities/PathUtility.cs`, `UOContent.csproj` |
| Deleted | `UOContent/Compression/TarArchive.cs`, `ZstdArchive.cs` |
| Tests | 29 new tests (ManagedArchive, ArchiveJournal, AutoArchiveHelpers) |

### Backward compatibility
- Existing `.tar.zst` archives are fully readable by the new managed code
- Config keys preserved; new keys use sensible defaults
- `autoArchive.compressionFormat` setting removed (Zstd only)
- Legacy `bsdtar/` directory logged as removable on startup

### Plugin example (external repo)
```csharp
public static class S3Plugin
{
    public static void Configure()
    {
        ArchiveDestinationRegistry.Register(new S3Destination("my-bucket", "us-east-1"));
    }
}
```

## Test plan
- [x] 29 new unit tests covering archive create/extract/count, journal state machine, recovery, destinations
- [x] All 969 tests pass (671 Server + 298 UOContent)
- [x] Manual: run server, trigger save, verify backup created and archive rolled up
- [x] Manual: kill server mid-archive, restart, verify journal recovery
- [x] Manual: test with large save files (~500MB+) to benchmark streaming vs old approach
2026-03-22 19:51:20 -07:00

230 lines
7 KiB
C#

using System;
using System.Collections.Generic;
using System.IO;
using Server.Saves;
using Xunit;
namespace Server.Tests;
[Collection("Sequential UOContent Tests")]
public class ArchiveJournalTests : IDisposable
{
private readonly string _testDir;
public ArchiveJournalTests()
{
_testDir = Path.Combine(Path.GetTempPath(), $"modernuo-journal-test-{Guid.NewGuid():N}");
Directory.CreateDirectory(_testDir);
}
public void Dispose()
{
if (Directory.Exists(_testDir))
{
Directory.Delete(_testDir, true);
}
}
[Fact]
public void Configure_LoadsCleanState()
{
// Arrange & Act
ArchiveJournal.Configure(_testDir);
// Assert - fresh directory should have no operations
// Note: ArchiveJournal is static, so we check the journal file doesn't exist
var journalPath = Path.Combine(_testDir, ".archive-journal.json");
Assert.False(File.Exists(journalPath));
}
[Fact]
public void BeginOperation_CreatesStartedEntry()
{
// Arrange
ArchiveJournal.Configure(_testDir);
// Act
var entry = ArchiveJournal.BeginOperation(
ArchivePeriod.Hourly,
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
"/tmp/archive.tar.zst.tmp",
"/archives/hourly/2026-03-22-14.tar.zst",
["backup1", "backup2"]
);
// Assert
Assert.Equal(ArchiveOperationState.Started, entry.State);
Assert.Equal(ArchivePeriod.Hourly, entry.Period);
Assert.Equal("/tmp/archive.tar.zst.tmp", entry.TempFile);
Assert.Equal(2, entry.SourceDirectories.Count);
Assert.Contains(entry, ArchiveJournal.Operations);
}
[Fact]
public void RecordArchived_TransitionsState()
{
// Arrange
ArchiveJournal.Configure(_testDir);
var entry = ArchiveJournal.BeginOperation(
ArchivePeriod.Daily,
new DateTime(2026, 3, 22, 0, 0, 0, DateTimeKind.Utc),
"/tmp/test.tmp",
"/archives/daily/2026-03-22.tar.zst",
["backup1"]
);
// Act
ArchiveJournal.RecordArchived(entry, 42, 1024000);
// Assert
Assert.Equal(ArchiveOperationState.Archived, entry.State);
Assert.Equal(42, entry.EntryCount);
Assert.Equal(1024000, entry.ArchiveBytes);
Assert.Null(entry.TempFile); // Temp file cleared after rename
}
[Fact]
public void RecordDistributed_StoresResults()
{
// Arrange
ArchiveJournal.Configure(_testDir);
var entry = ArchiveJournal.BeginOperation(
ArchivePeriod.Monthly,
new DateTime(2026, 3, 1, 0, 0, 0, DateTimeKind.Utc),
"/tmp/test.tmp",
"/archives/monthly/2026-03.tar.zst",
["backup1"]
);
ArchiveJournal.RecordArchived(entry, 100, 5000000);
var results = new Dictionary<string, bool>
{
{ "Local Filesystem", true },
{ "S3 us-east-1", false }
};
// Act
ArchiveJournal.RecordDistributed(entry, results);
// Assert
Assert.Equal(ArchiveOperationState.Distributed, entry.State);
Assert.True(entry.DestinationResults["Local Filesystem"]);
Assert.False(entry.DestinationResults["S3 us-east-1"]);
}
[Fact]
public void RecordCompleted_IsTerminal()
{
// Arrange
ArchiveJournal.Configure(_testDir);
var entry = ArchiveJournal.BeginOperation(
ArchivePeriod.Hourly,
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
"/tmp/test.tmp",
"/archives/hourly/test.tar.zst",
["backup1"]
);
ArchiveJournal.RecordArchived(entry, 10, 500);
ArchiveJournal.RecordDistributed(entry, new Dictionary<string, bool>());
// Act
ArchiveJournal.RecordCompleted(entry);
// Assert
Assert.Equal(ArchiveOperationState.Completed, entry.State);
Assert.NotEqual(DateTime.MaxValue, entry.CompletedAt);
}
[Fact]
public void RecordFailure_RecordsReason()
{
// Arrange
ArchiveJournal.Configure(_testDir);
var entry = ArchiveJournal.BeginOperation(
ArchivePeriod.Hourly,
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
"/tmp/test.tmp",
"/archives/hourly/test.tar.zst",
["backup1"]
);
// Act
ArchiveJournal.RecordFailure(entry, "Disk full");
// Assert
Assert.Equal(ArchiveOperationState.Failed, entry.State);
Assert.Equal("Disk full", entry.FailureReason);
Assert.NotEqual(DateTime.MaxValue, entry.CompletedAt);
}
[Fact]
public void Journal_PersistsToJsonFile()
{
// Arrange
ArchiveJournal.Configure(_testDir);
ArchiveJournal.BeginOperation(
ArchivePeriod.Hourly,
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
"/tmp/test.tmp",
"/archives/hourly/test.tar.zst",
["backup1"]
);
// Assert - journal file should exist
var journalPath = Path.Combine(_testDir, ".archive-journal.json");
Assert.True(File.Exists(journalPath));
var content = File.ReadAllText(journalPath);
Assert.Contains("Hourly", content);
Assert.Contains("Started", content);
}
[Fact]
public void RecoverInterrupted_CleansUpStartedOperations()
{
// Arrange
ArchiveJournal.Configure(_testDir);
// Create a temp file that simulates an interrupted archive
var tempFile = Path.Combine(_testDir, "interrupted.tar.zst.tmp");
File.WriteAllText(tempFile, "partial archive data");
var entry = ArchiveJournal.BeginOperation(
ArchivePeriod.Hourly,
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
tempFile,
Path.Combine(_testDir, "final.tar.zst"),
["backup1"]
);
// Act
ArchiveJournal.RecoverInterrupted();
// Assert
Assert.Equal(ArchiveOperationState.Failed, entry.State);
Assert.Equal("Interrupted during archive creation", entry.FailureReason);
Assert.False(File.Exists(tempFile)); // Temp file should be cleaned up
}
[Fact]
public void RecoverInterrupted_CompletesDistributedOperations()
{
// Arrange
ArchiveJournal.Configure(_testDir);
var entry = ArchiveJournal.BeginOperation(
ArchivePeriod.Daily,
new DateTime(2026, 3, 22, 0, 0, 0, DateTimeKind.Utc),
"/tmp/test.tmp",
"/archives/daily/test.tar.zst",
["backup1"]
);
ArchiveJournal.RecordArchived(entry, 50, 2000);
ArchiveJournal.RecordDistributed(entry, new Dictionary<string, bool> { { "Local", true } });
// Act - simulate restart
ArchiveJournal.RecoverInterrupted();
// Assert - should be marked completed
Assert.Equal(ArchiveOperationState.Completed, entry.State);
}
}