feat: Implements new robust/pluggable backup/archive system. (#2388)
## Summary
Overhauls the backup/archive system to address stability, robustness, fault tolerance, performance, and pluggability.
### Problems solved
- **Critical bug**: `Interlocked.CompareExchange` arguments were reversed — the concurrent archive guard never actually prevented concurrent archives
- **Brittle compression**: Relied on spawning external `zstd.exe`/`tar.exe`/`bsdtar.exe` processes with platform-specific workarounds (Windows two-step hack, auto-downloading bsdtar from libarchive.org)
- **No fault tolerance**: Partial archives possible if process killed mid-write; no recovery mechanism; originals deleted before verification
- **No extensibility**: No way to add remote backup destinations (S3, rsync) without modifying core code
- **Silent failures**: Bare `catch` blocks swallowed exceptions with no diagnostics
### What changed
**Cross-platform streaming compression** — Replaced external process spawning with `System.Formats.Tar` (built-in .NET) + `ZstdNet` (native libzstd P/Invoke). Single-pass streaming: `TarWriter → CompressionStream → FileStream`. No intermediate `.tar` file, no platform-specific code paths.
**Archive journal** — JSON-based operation journal (`Archives/.archive-journal.json`) tracks state machine: `Started → Archived → Distributed → Completed` (or `Failed`). On startup, recovers interrupted operations (cleans temp files, completes pruning).
**Verify before delete** — Archives are verified (entry count check) before deleting source backups. Temp-file-then-atomic-rename pattern prevents partial archives.
**Retry logic** — File operations (moves, deletes) retry with linear backoff for transient I/O failures (antivirus locks, file copy operations).
**Plugin architecture** — `IArchiveDestination` interface in `Projects/Server/` enables external plugins (loaded via `assemblies.json`) to receive completed archives. `ArchiveDestinationRegistry` tracks destinations. Conservative pruning: source backups preserved if any destination with retention fails.
**Configurable** — Retention counts (hourly/daily/monthly), compression level, retry settings, backup max age all configurable via `ServerConfiguration`.
### New admin commands
- `[ArchiveStatus` — Shows journal state, destinations, next scheduled archive times
- `[ArchiveNow` — Forces immediate rollup regardless of schedule
### Files
| Change | File |
|--------|------|
| New | `Server/Saves/ArchiveEnums.cs`, `IArchiveDestination.cs`, `ArchiveDestinationRegistry.cs`, `ArchiveEventArgs.cs` |
| New | `Server/Events/ArchiveEvents.cs` (partial EventSink) |
| New | `UOContent/Compression/ManagedArchive.cs` |
| New | `UOContent/World Saves/ArchiveJournal.cs`, `LocalArchiveDestination.cs` |
| Rewrite | `UOContent/World Saves/AutoArchive.cs` |
| Modified | `UOContent/World Saves/SaveCommands.cs`, `Server/Utilities/PathUtility.cs`, `UOContent.csproj` |
| Deleted | `UOContent/Compression/TarArchive.cs`, `ZstdArchive.cs` |
| Tests | 29 new tests (ManagedArchive, ArchiveJournal, AutoArchiveHelpers) |
### Backward compatibility
- Existing `.tar.zst` archives are fully readable by the new managed code
- Config keys preserved; new keys use sensible defaults
- `autoArchive.compressionFormat` setting removed (Zstd only)
- Legacy `bsdtar/` directory logged as removable on startup
### Plugin example (external repo)
```csharp
public static class S3Plugin
{
public static void Configure()
{
ArchiveDestinationRegistry.Register(new S3Destination("my-bucket", "us-east-1"));
}
}
```
## Test plan
- [x] 29 new unit tests covering archive create/extract/count, journal state machine, recovery, destinations
- [x] All 969 tests pass (671 Server + 298 UOContent)
- [x] Manual: run server, trigger save, verify backup created and archive rolled up
- [x] Manual: kill server mid-archive, restart, verify journal recovery
- [x] Manual: test with large save files (~500MB+) to benchmark streaming vs old approach
This commit is contained in:
parent
61e41df00c
commit
3b4e1137f6
15 changed files with 2421 additions and 615 deletions
230
Projects/UOContent.Tests/Tests/WorldSaves/ArchiveJournalTests.cs
Normal file
230
Projects/UOContent.Tests/Tests/WorldSaves/ArchiveJournalTests.cs
Normal file
|
|
@ -0,0 +1,230 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using Server.Saves;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace Server.Tests;
|
||||||
|
|
||||||
|
[Collection("Sequential UOContent Tests")]
|
||||||
|
public class ArchiveJournalTests : IDisposable
|
||||||
|
{
|
||||||
|
private readonly string _testDir;
|
||||||
|
|
||||||
|
public ArchiveJournalTests()
|
||||||
|
{
|
||||||
|
_testDir = Path.Combine(Path.GetTempPath(), $"modernuo-journal-test-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(_testDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
if (Directory.Exists(_testDir))
|
||||||
|
{
|
||||||
|
Directory.Delete(_testDir, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Configure_LoadsCleanState()
|
||||||
|
{
|
||||||
|
// Arrange & Act
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
|
||||||
|
// Assert - fresh directory should have no operations
|
||||||
|
// Note: ArchiveJournal is static, so we check the journal file doesn't exist
|
||||||
|
var journalPath = Path.Combine(_testDir, ".archive-journal.json");
|
||||||
|
Assert.False(File.Exists(journalPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void BeginOperation_CreatesStartedEntry()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var entry = ArchiveJournal.BeginOperation(
|
||||||
|
ArchivePeriod.Hourly,
|
||||||
|
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
|
||||||
|
"/tmp/archive.tar.zst.tmp",
|
||||||
|
"/archives/hourly/2026-03-22-14.tar.zst",
|
||||||
|
["backup1", "backup2"]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(ArchiveOperationState.Started, entry.State);
|
||||||
|
Assert.Equal(ArchivePeriod.Hourly, entry.Period);
|
||||||
|
Assert.Equal("/tmp/archive.tar.zst.tmp", entry.TempFile);
|
||||||
|
Assert.Equal(2, entry.SourceDirectories.Count);
|
||||||
|
Assert.Contains(entry, ArchiveJournal.Operations);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordArchived_TransitionsState()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
var entry = ArchiveJournal.BeginOperation(
|
||||||
|
ArchivePeriod.Daily,
|
||||||
|
new DateTime(2026, 3, 22, 0, 0, 0, DateTimeKind.Utc),
|
||||||
|
"/tmp/test.tmp",
|
||||||
|
"/archives/daily/2026-03-22.tar.zst",
|
||||||
|
["backup1"]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
ArchiveJournal.RecordArchived(entry, 42, 1024000);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(ArchiveOperationState.Archived, entry.State);
|
||||||
|
Assert.Equal(42, entry.EntryCount);
|
||||||
|
Assert.Equal(1024000, entry.ArchiveBytes);
|
||||||
|
Assert.Null(entry.TempFile); // Temp file cleared after rename
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordDistributed_StoresResults()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
var entry = ArchiveJournal.BeginOperation(
|
||||||
|
ArchivePeriod.Monthly,
|
||||||
|
new DateTime(2026, 3, 1, 0, 0, 0, DateTimeKind.Utc),
|
||||||
|
"/tmp/test.tmp",
|
||||||
|
"/archives/monthly/2026-03.tar.zst",
|
||||||
|
["backup1"]
|
||||||
|
);
|
||||||
|
ArchiveJournal.RecordArchived(entry, 100, 5000000);
|
||||||
|
|
||||||
|
var results = new Dictionary<string, bool>
|
||||||
|
{
|
||||||
|
{ "Local Filesystem", true },
|
||||||
|
{ "S3 us-east-1", false }
|
||||||
|
};
|
||||||
|
|
||||||
|
// Act
|
||||||
|
ArchiveJournal.RecordDistributed(entry, results);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(ArchiveOperationState.Distributed, entry.State);
|
||||||
|
Assert.True(entry.DestinationResults["Local Filesystem"]);
|
||||||
|
Assert.False(entry.DestinationResults["S3 us-east-1"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordCompleted_IsTerminal()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
var entry = ArchiveJournal.BeginOperation(
|
||||||
|
ArchivePeriod.Hourly,
|
||||||
|
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
|
||||||
|
"/tmp/test.tmp",
|
||||||
|
"/archives/hourly/test.tar.zst",
|
||||||
|
["backup1"]
|
||||||
|
);
|
||||||
|
ArchiveJournal.RecordArchived(entry, 10, 500);
|
||||||
|
ArchiveJournal.RecordDistributed(entry, new Dictionary<string, bool>());
|
||||||
|
|
||||||
|
// Act
|
||||||
|
ArchiveJournal.RecordCompleted(entry);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(ArchiveOperationState.Completed, entry.State);
|
||||||
|
Assert.NotEqual(DateTime.MaxValue, entry.CompletedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordFailure_RecordsReason()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
var entry = ArchiveJournal.BeginOperation(
|
||||||
|
ArchivePeriod.Hourly,
|
||||||
|
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
|
||||||
|
"/tmp/test.tmp",
|
||||||
|
"/archives/hourly/test.tar.zst",
|
||||||
|
["backup1"]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
ArchiveJournal.RecordFailure(entry, "Disk full");
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(ArchiveOperationState.Failed, entry.State);
|
||||||
|
Assert.Equal("Disk full", entry.FailureReason);
|
||||||
|
Assert.NotEqual(DateTime.MaxValue, entry.CompletedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Journal_PersistsToJsonFile()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
ArchiveJournal.BeginOperation(
|
||||||
|
ArchivePeriod.Hourly,
|
||||||
|
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
|
||||||
|
"/tmp/test.tmp",
|
||||||
|
"/archives/hourly/test.tar.zst",
|
||||||
|
["backup1"]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert - journal file should exist
|
||||||
|
var journalPath = Path.Combine(_testDir, ".archive-journal.json");
|
||||||
|
Assert.True(File.Exists(journalPath));
|
||||||
|
|
||||||
|
var content = File.ReadAllText(journalPath);
|
||||||
|
Assert.Contains("Hourly", content);
|
||||||
|
Assert.Contains("Started", content);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecoverInterrupted_CleansUpStartedOperations()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
|
||||||
|
// Create a temp file that simulates an interrupted archive
|
||||||
|
var tempFile = Path.Combine(_testDir, "interrupted.tar.zst.tmp");
|
||||||
|
File.WriteAllText(tempFile, "partial archive data");
|
||||||
|
|
||||||
|
var entry = ArchiveJournal.BeginOperation(
|
||||||
|
ArchivePeriod.Hourly,
|
||||||
|
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
|
||||||
|
tempFile,
|
||||||
|
Path.Combine(_testDir, "final.tar.zst"),
|
||||||
|
["backup1"]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
ArchiveJournal.RecoverInterrupted();
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(ArchiveOperationState.Failed, entry.State);
|
||||||
|
Assert.Equal("Interrupted during archive creation", entry.FailureReason);
|
||||||
|
Assert.False(File.Exists(tempFile)); // Temp file should be cleaned up
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecoverInterrupted_CompletesDistributedOperations()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
ArchiveJournal.Configure(_testDir);
|
||||||
|
var entry = ArchiveJournal.BeginOperation(
|
||||||
|
ArchivePeriod.Daily,
|
||||||
|
new DateTime(2026, 3, 22, 0, 0, 0, DateTimeKind.Utc),
|
||||||
|
"/tmp/test.tmp",
|
||||||
|
"/archives/daily/test.tar.zst",
|
||||||
|
["backup1"]
|
||||||
|
);
|
||||||
|
ArchiveJournal.RecordArchived(entry, 50, 2000);
|
||||||
|
ArchiveJournal.RecordDistributed(entry, new Dictionary<string, bool> { { "Local", true } });
|
||||||
|
|
||||||
|
// Act - simulate restart
|
||||||
|
ArchiveJournal.RecoverInterrupted();
|
||||||
|
|
||||||
|
// Assert - should be marked completed
|
||||||
|
Assert.Equal(ArchiveOperationState.Completed, entry.State);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
using System;
|
||||||
|
using Server.Saves;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace Server.Tests;
|
||||||
|
|
||||||
|
[Collection("Sequential UOContent Tests")]
|
||||||
|
public class AutoArchiveHelperTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void ArchiveDestinationRegistry_RegisterAndRetrieve()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var dest = new LocalArchiveDestination(24, 30, 12);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
ArchiveDestinationRegistry.Register(dest);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Contains(dest, ArchiveDestinationRegistry.Destinations);
|
||||||
|
|
||||||
|
// Cleanup
|
||||||
|
ArchiveDestinationRegistry.Unregister(dest);
|
||||||
|
Assert.DoesNotContain(dest, ArchiveDestinationRegistry.Destinations);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void LocalArchiveDestination_SendArchive_ReturnsTrue()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var dest = new LocalArchiveDestination(24, 30, 12);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var result = dest.SendArchive("/fake/path.tar.zst", ArchivePeriod.Hourly, DateTime.UtcNow);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.True(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData(ArchivePeriod.Hourly, 24)]
|
||||||
|
[InlineData(ArchivePeriod.Daily, 30)]
|
||||||
|
[InlineData(ArchivePeriod.Monthly, 12)]
|
||||||
|
public void LocalArchiveDestination_RetentionCounts(ArchivePeriod period, int expected)
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var dest = new LocalArchiveDestination(24, 30, 12);
|
||||||
|
|
||||||
|
// Act & Assert
|
||||||
|
Assert.Equal(expected, dest.GetRetentionCount(period));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ArchiveCompletedEventArgs_StoresValues()
|
||||||
|
{
|
||||||
|
// Arrange & Act
|
||||||
|
var args = new ArchiveCompletedEventArgs(
|
||||||
|
"/archives/hourly/test.tar.zst",
|
||||||
|
ArchivePeriod.Hourly,
|
||||||
|
new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc),
|
||||||
|
1048576,
|
||||||
|
2.5
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal("/archives/hourly/test.tar.zst", args.ArchiveFilePath);
|
||||||
|
Assert.Equal(ArchivePeriod.Hourly, args.Period);
|
||||||
|
Assert.Equal(1048576, args.FileSizeBytes);
|
||||||
|
Assert.Equal(2.5, args.ElapsedSeconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ArchiveFailedEventArgs_StoresException()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var ex = new InvalidOperationException("disk full");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var args = new ArchiveFailedEventArgs(
|
||||||
|
ArchivePeriod.Daily,
|
||||||
|
new DateTime(2026, 3, 22, 0, 0, 0, DateTimeKind.Utc),
|
||||||
|
ex
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(ArchivePeriod.Daily, args.Period);
|
||||||
|
Assert.Same(ex, args.Exception);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ArchiveJournalEntry_DefaultState()
|
||||||
|
{
|
||||||
|
// Arrange & Act
|
||||||
|
var entry = new ArchiveJournalEntry();
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.NotNull(entry.SourceDirectories);
|
||||||
|
Assert.Empty(entry.SourceDirectories);
|
||||||
|
Assert.NotNull(entry.DestinationResults);
|
||||||
|
Assert.Empty(entry.DestinationResults);
|
||||||
|
Assert.Null(entry.FailureReason);
|
||||||
|
Assert.Equal(DateTime.MaxValue, entry.CompletedAt);
|
||||||
|
}
|
||||||
|
}
|
||||||
325
Projects/UOContent.Tests/Tests/WorldSaves/ManagedArchiveTests.cs
Normal file
325
Projects/UOContent.Tests/Tests/WorldSaves/ManagedArchiveTests.cs
Normal file
|
|
@ -0,0 +1,325 @@
|
||||||
|
using System;
|
||||||
|
using System.Formats.Tar;
|
||||||
|
using System.IO;
|
||||||
|
using System.IO.Compression;
|
||||||
|
using Server.Compression;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace Server.Tests;
|
||||||
|
|
||||||
|
[Collection("Sequential UOContent Tests")]
|
||||||
|
public class ManagedArchiveTests : IDisposable
|
||||||
|
{
|
||||||
|
private readonly string _testDir;
|
||||||
|
|
||||||
|
public ManagedArchiveTests()
|
||||||
|
{
|
||||||
|
_testDir = Path.Combine(Path.GetTempPath(), $"modernuo-test-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(_testDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
if (Directory.Exists(_testDir))
|
||||||
|
{
|
||||||
|
Directory.Delete(_testDir, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private string CreateTestDirectory(string name, params (string fileName, string content)[] files)
|
||||||
|
{
|
||||||
|
var dir = Path.Combine(_testDir, name);
|
||||||
|
Directory.CreateDirectory(dir);
|
||||||
|
|
||||||
|
foreach (var (fileName, content) in files)
|
||||||
|
{
|
||||||
|
var filePath = Path.Combine(dir, fileName);
|
||||||
|
var fileDir = Path.GetDirectoryName(filePath);
|
||||||
|
if (fileDir != null)
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(fileDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
File.WriteAllText(filePath, content);
|
||||||
|
}
|
||||||
|
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CreateTarZstd_SingleDirectory_RoundTrips()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var sourceDir = CreateTestDirectory("source",
|
||||||
|
("file1.txt", "Hello World"),
|
||||||
|
("file2.bin", "Binary content here"),
|
||||||
|
("subdir/nested.txt", "Nested file")
|
||||||
|
);
|
||||||
|
|
||||||
|
var archivePath = Path.Combine(_testDir, "test.tar.zst");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var entryCount = ManagedArchive.CreateTarZstd(
|
||||||
|
[sourceDir], archivePath, _testDir, compressionLevel: 1
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.True(entryCount > 0);
|
||||||
|
Assert.True(File.Exists(archivePath));
|
||||||
|
Assert.True(new FileInfo(archivePath).Length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CreateTarZstd_MultipleDirectories_IncludesAll()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var dir1 = CreateTestDirectory("backup1",
|
||||||
|
("items.bin", "items data"),
|
||||||
|
("items.idx", "items index")
|
||||||
|
);
|
||||||
|
var dir2 = CreateTestDirectory("backup2",
|
||||||
|
("mobiles.bin", "mobiles data"),
|
||||||
|
("mobiles.idx", "mobiles index")
|
||||||
|
);
|
||||||
|
|
||||||
|
var archivePath = Path.Combine(_testDir, "multi.tar.zst");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var entryCount = ManagedArchive.CreateTarZstd(
|
||||||
|
[dir1, dir2], archivePath, _testDir, compressionLevel: 1
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(4, entryCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ExtractTarZstd_RecreatesFiles()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var sourceDir = CreateTestDirectory("source",
|
||||||
|
("data.txt", "Test data content"),
|
||||||
|
("config/settings.json", "{\"key\":\"value\"}")
|
||||||
|
);
|
||||||
|
|
||||||
|
var archivePath = Path.Combine(_testDir, "extract-test.tar.zst");
|
||||||
|
ManagedArchive.CreateTarZstd([sourceDir], archivePath, _testDir, compressionLevel: 1);
|
||||||
|
|
||||||
|
var extractDir = Path.Combine(_testDir, "extracted");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var success = ManagedArchive.ExtractTarZstd(archivePath, extractDir);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.True(success);
|
||||||
|
Assert.True(Directory.Exists(extractDir));
|
||||||
|
|
||||||
|
// Verify extracted content matches
|
||||||
|
var extractedDataPath = Path.Combine(extractDir, "source", "data.txt");
|
||||||
|
Assert.True(File.Exists(extractedDataPath));
|
||||||
|
Assert.Equal("Test data content", File.ReadAllText(extractedDataPath));
|
||||||
|
|
||||||
|
var extractedConfigPath = Path.Combine(extractDir, "source", "config", "settings.json");
|
||||||
|
Assert.True(File.Exists(extractedConfigPath));
|
||||||
|
Assert.Equal("{\"key\":\"value\"}", File.ReadAllText(extractedConfigPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CountEntries_MatchesCreateCount()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var sourceDir = CreateTestDirectory("source",
|
||||||
|
("a.txt", "a"),
|
||||||
|
("b.txt", "b"),
|
||||||
|
("c/d.txt", "d"),
|
||||||
|
("c/e.txt", "e")
|
||||||
|
);
|
||||||
|
|
||||||
|
var archivePath = Path.Combine(_testDir, "count-test.tar.zst");
|
||||||
|
var createCount = ManagedArchive.CreateTarZstd(
|
||||||
|
[sourceDir], archivePath, _testDir, compressionLevel: 1
|
||||||
|
);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var verifyCount = ManagedArchive.CountEntries(archivePath);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(createCount, verifyCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CreateTarZstd_EmptyDirectory_ReturnsZero()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var sourceDir = Path.Combine(_testDir, "empty");
|
||||||
|
Directory.CreateDirectory(sourceDir);
|
||||||
|
|
||||||
|
var archivePath = Path.Combine(_testDir, "empty.tar.zst");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var entryCount = ManagedArchive.CreateTarZstd(
|
||||||
|
[sourceDir], archivePath, _testDir, compressionLevel: 1
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(0, entryCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CreateTarZstd_NonexistentSource_SkipsGracefully()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var archivePath = Path.Combine(_testDir, "nonexistent.tar.zst");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var entryCount = ManagedArchive.CreateTarZstd(
|
||||||
|
[Path.Combine(_testDir, "does-not-exist")], archivePath, _testDir, compressionLevel: 1
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(0, entryCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CreateTarZstd_TempFileCleanedUp()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var sourceDir = CreateTestDirectory("source", ("file.txt", "content"));
|
||||||
|
var archivePath = Path.Combine(_testDir, "cleanup-test.tar.zst");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
ManagedArchive.CreateTarZstd([sourceDir], archivePath, _testDir, compressionLevel: 1);
|
||||||
|
|
||||||
|
// Assert - no .tmp file should remain
|
||||||
|
Assert.False(File.Exists(archivePath + ".tmp"));
|
||||||
|
Assert.True(File.Exists(archivePath));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ExtractTarZstd_InvalidFile_ReturnsFalse()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var badFile = Path.Combine(_testDir, "bad.tar.zst");
|
||||||
|
File.WriteAllText(badFile, "this is not a valid archive");
|
||||||
|
var extractDir = Path.Combine(_testDir, "extract-bad");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var result = ManagedArchive.ExtractTarZstd(badFile, extractDir);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.False(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CountEntries_InvalidFile_ReturnsNegative()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var badFile = Path.Combine(_testDir, "bad.tar.zst");
|
||||||
|
File.WriteAllText(badFile, "not a valid archive");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var count = ManagedArchive.CountEntries(badFile);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(-1, count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData(1)]
|
||||||
|
[InlineData(3)]
|
||||||
|
[InlineData(10)]
|
||||||
|
public void CreateTarZstd_DifferentCompressionLevels_AllWork(int level)
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var sourceDir = CreateTestDirectory($"level-{level}", ("data.txt", new string('X', 10000)));
|
||||||
|
var archivePath = Path.Combine(_testDir, $"level-{level}.tar.zst");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var entryCount = ManagedArchive.CreateTarZstd(
|
||||||
|
[sourceDir], archivePath, _testDir, compressionLevel: level
|
||||||
|
);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.Equal(1, entryCount);
|
||||||
|
Assert.True(File.Exists(archivePath));
|
||||||
|
|
||||||
|
// Verify we can extract it back
|
||||||
|
var extractDir = Path.Combine(_testDir, $"extract-level-{level}");
|
||||||
|
Assert.True(ManagedArchive.ExtractTarZstd(archivePath, extractDir));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ExtractTarGz_RoundTrips()
|
||||||
|
{
|
||||||
|
// Arrange — create a .tar.gz using built-in .NET APIs
|
||||||
|
var sourceDir = CreateTestDirectory("gz-source",
|
||||||
|
("data.txt", "GZip test data"),
|
||||||
|
("sub/nested.txt", "Nested content")
|
||||||
|
);
|
||||||
|
|
||||||
|
var archivePath = Path.Combine(_testDir, "test.tar.gz");
|
||||||
|
|
||||||
|
using (var fileStream = new FileStream(archivePath, FileMode.Create))
|
||||||
|
using (var gzStream = new GZipStream(fileStream, System.IO.Compression.CompressionLevel.Fastest))
|
||||||
|
using (var tarWriter = new TarWriter(gzStream, TarEntryFormat.Pax, leaveOpen: true))
|
||||||
|
{
|
||||||
|
foreach (var file in new DirectoryInfo(sourceDir).EnumerateFiles("*", SearchOption.AllDirectories))
|
||||||
|
{
|
||||||
|
var entryName = Path.GetRelativePath(sourceDir, file.FullName).Replace('\\', '/');
|
||||||
|
tarWriter.WriteEntry(file.FullName, entryName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var extractDir = Path.Combine(_testDir, "gz-extracted");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var success = ManagedArchive.ExtractTarGz(archivePath, extractDir);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.True(success);
|
||||||
|
Assert.Equal("GZip test data", File.ReadAllText(Path.Combine(extractDir, "data.txt")));
|
||||||
|
Assert.Equal("Nested content", File.ReadAllText(Path.Combine(extractDir, "sub", "nested.txt")));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ExtractTar_PlainTarRoundTrips()
|
||||||
|
{
|
||||||
|
// Arrange — create a plain .tar using built-in .NET APIs
|
||||||
|
var sourceDir = CreateTestDirectory("tar-source",
|
||||||
|
("plain.txt", "Plain tar test data")
|
||||||
|
);
|
||||||
|
|
||||||
|
var archivePath = Path.Combine(_testDir, "test.tar");
|
||||||
|
|
||||||
|
using (var fileStream = new FileStream(archivePath, FileMode.Create))
|
||||||
|
using (var tarWriter = new TarWriter(fileStream, TarEntryFormat.Pax, leaveOpen: true))
|
||||||
|
{
|
||||||
|
foreach (var file in new DirectoryInfo(sourceDir).EnumerateFiles("*", SearchOption.AllDirectories))
|
||||||
|
{
|
||||||
|
var entryName = Path.GetRelativePath(sourceDir, file.FullName).Replace('\\', '/');
|
||||||
|
tarWriter.WriteEntry(file.FullName, entryName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var extractDir = Path.Combine(_testDir, "tar-extracted");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var success = ManagedArchive.ExtractTar(archivePath, extractDir);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
Assert.True(success);
|
||||||
|
Assert.Equal("Plain tar test data", File.ReadAllText(Path.Combine(extractDir, "plain.txt")));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ExtractTarGz_InvalidFile_ReturnsFalse()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var badFile = Path.Combine(_testDir, "bad.tar.gz");
|
||||||
|
File.WriteAllText(badFile, "not a valid gzip archive");
|
||||||
|
var extractDir = Path.Combine(_testDir, "extract-bad-gz");
|
||||||
|
|
||||||
|
// Act & Assert
|
||||||
|
Assert.False(ManagedArchive.ExtractTarGz(badFile, extractDir));
|
||||||
|
}
|
||||||
|
}
|
||||||
279
Projects/UOContent/Compression/ManagedArchive.cs
Normal file
279
Projects/UOContent/Compression/ManagedArchive.cs
Normal file
|
|
@ -0,0 +1,279 @@
|
||||||
|
/*************************************************************************
|
||||||
|
* ModernUO *
|
||||||
|
* Copyright 2019-2026 - ModernUO Development Team *
|
||||||
|
* Email: hi@modernuo.com *
|
||||||
|
* File: ManagedArchive.cs *
|
||||||
|
* *
|
||||||
|
* This program is free software: you can redistribute it and/or modify *
|
||||||
|
* it under the terms of the GNU General Public License as published by *
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or *
|
||||||
|
* (at your option) any later version. *
|
||||||
|
* *
|
||||||
|
* You should have received a copy of the GNU General Public License *
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||||
|
*************************************************************************/
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Formats.Tar;
|
||||||
|
using System.IO;
|
||||||
|
using System.IO.Compression;
|
||||||
|
using Server.Logging;
|
||||||
|
using ZstdNet;
|
||||||
|
|
||||||
|
namespace Server.Compression;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Cross-platform archive creation and extraction using System.Formats.Tar + ZstdNet (native libzstd).
|
||||||
|
/// Single-pass streaming: TarWriter → CompressionStream → FileStream.
|
||||||
|
/// </summary>
|
||||||
|
public static class ManagedArchive
|
||||||
|
{
|
||||||
|
private static readonly ILogger logger = LogFactory.GetLogger(typeof(ManagedArchive));
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a .tar.zst archive from one or more source directories.
|
||||||
|
/// Writes to a temp file first, then atomically renames to the destination.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="sourcePaths">Directories to include in the archive.</param>
|
||||||
|
/// <param name="destinationFile">Final .tar.zst file path.</param>
|
||||||
|
/// <param name="relativeTo">Base path for computing relative entry names.</param>
|
||||||
|
/// <param name="compressionLevel">ZSTD compression level (1-22, default 3).</param>
|
||||||
|
/// <returns>The number of entries written, or -1 on failure.</returns>
|
||||||
|
public static int CreateTarZstd(
|
||||||
|
IEnumerable<string> sourcePaths,
|
||||||
|
string destinationFile,
|
||||||
|
string relativeTo,
|
||||||
|
int compressionLevel = 3)
|
||||||
|
{
|
||||||
|
var tempFile = destinationFile + ".tmp";
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
new FileInfo(destinationFile).EnsureDirectory();
|
||||||
|
|
||||||
|
using var fileStream = new FileStream(
|
||||||
|
tempFile,
|
||||||
|
FileMode.Create,
|
||||||
|
FileAccess.Write,
|
||||||
|
FileShare.None,
|
||||||
|
bufferSize: 81920
|
||||||
|
);
|
||||||
|
|
||||||
|
using var compressionOptions = new CompressionOptions(compressionLevel);
|
||||||
|
using var zstdStream = new CompressionStream(fileStream, compressionOptions);
|
||||||
|
using var tarWriter = new TarWriter(zstdStream, TarEntryFormat.Pax, leaveOpen: true);
|
||||||
|
|
||||||
|
var entryCount = 0;
|
||||||
|
long totalBytes = 0;
|
||||||
|
|
||||||
|
foreach (var sourcePath in sourcePaths)
|
||||||
|
{
|
||||||
|
if (!Directory.Exists(sourcePath))
|
||||||
|
{
|
||||||
|
logger.Warning("Source path does not exist, skipping: {Path}", sourcePath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sourceDir = new DirectoryInfo(sourcePath);
|
||||||
|
var relativeName = Path.GetRelativePath(relativeTo, sourcePath);
|
||||||
|
|
||||||
|
foreach (var file in sourceDir.EnumerateFiles("*", SearchOption.AllDirectories))
|
||||||
|
{
|
||||||
|
var entryName = Path.Combine(relativeName, Path.GetRelativePath(sourcePath, file.FullName))
|
||||||
|
.Replace('\\', '/');
|
||||||
|
|
||||||
|
tarWriter.WriteEntry(file.FullName, entryName);
|
||||||
|
entryCount++;
|
||||||
|
totalBytes += file.Length;
|
||||||
|
|
||||||
|
if (entryCount % 100 == 0)
|
||||||
|
{
|
||||||
|
logger.Debug(
|
||||||
|
"Archive progress: {Count} entries, {Size:F1} MB",
|
||||||
|
entryCount,
|
||||||
|
totalBytes / 1048576.0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush tar → zstd → file
|
||||||
|
tarWriter.Dispose();
|
||||||
|
zstdStream.Dispose();
|
||||||
|
fileStream.Dispose();
|
||||||
|
|
||||||
|
// Atomic rename on same filesystem
|
||||||
|
if (File.Exists(destinationFile))
|
||||||
|
{
|
||||||
|
File.Delete(destinationFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
File.Move(tempFile, destinationFile);
|
||||||
|
|
||||||
|
logger.Information(
|
||||||
|
"Created archive at {Path} ({Count} entries, {Size:F1} MB)",
|
||||||
|
destinationFile,
|
||||||
|
entryCount,
|
||||||
|
totalBytes / 1048576.0
|
||||||
|
);
|
||||||
|
|
||||||
|
return entryCount;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger.Error(ex, "Failed to create archive at {Path}", destinationFile);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (File.Exists(tempFile))
|
||||||
|
{
|
||||||
|
File.Delete(tempFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Best-effort cleanup
|
||||||
|
}
|
||||||
|
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Extracts a .tar.zst archive to a directory.
|
||||||
|
/// </summary>
|
||||||
|
public static bool ExtractTarZstd(string archiveFile, string outputDirectory)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(outputDirectory);
|
||||||
|
|
||||||
|
using var fileStream = new FileStream(
|
||||||
|
archiveFile,
|
||||||
|
FileMode.Open,
|
||||||
|
FileAccess.Read,
|
||||||
|
FileShare.Read,
|
||||||
|
bufferSize: 81920
|
||||||
|
);
|
||||||
|
|
||||||
|
using var zstdStream = new DecompressionStream(fileStream);
|
||||||
|
return ExtractTarFromStream(zstdStream, outputDirectory);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger.Error(ex, "Failed to extract archive {File}", archiveFile);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Extracts a .tar.gz archive to a directory.
|
||||||
|
/// </summary>
|
||||||
|
public static bool ExtractTarGz(string archiveFile, string outputDirectory)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(outputDirectory);
|
||||||
|
|
||||||
|
using var fileStream = new FileStream(
|
||||||
|
archiveFile,
|
||||||
|
FileMode.Open,
|
||||||
|
FileAccess.Read,
|
||||||
|
FileShare.Read,
|
||||||
|
bufferSize: 81920
|
||||||
|
);
|
||||||
|
|
||||||
|
using var gzStream = new GZipStream(fileStream, CompressionMode.Decompress);
|
||||||
|
return ExtractTarFromStream(gzStream, outputDirectory);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger.Error(ex, "Failed to extract archive {File}", archiveFile);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Extracts a plain .tar archive to a directory.
|
||||||
|
/// </summary>
|
||||||
|
public static bool ExtractTar(string archiveFile, string outputDirectory)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(outputDirectory);
|
||||||
|
|
||||||
|
using var fileStream = new FileStream(
|
||||||
|
archiveFile,
|
||||||
|
FileMode.Open,
|
||||||
|
FileAccess.Read,
|
||||||
|
FileShare.Read,
|
||||||
|
bufferSize: 81920
|
||||||
|
);
|
||||||
|
|
||||||
|
return ExtractTarFromStream(fileStream, outputDirectory);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger.Error(ex, "Failed to extract archive {File}", archiveFile);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool ExtractTarFromStream(Stream stream, string outputDirectory)
|
||||||
|
{
|
||||||
|
using var tarReader = new TarReader(stream);
|
||||||
|
|
||||||
|
while (tarReader.GetNextEntry() is { } entry)
|
||||||
|
{
|
||||||
|
var destPath = Path.Combine(outputDirectory, entry.Name);
|
||||||
|
|
||||||
|
var destDir = Path.GetDirectoryName(destPath);
|
||||||
|
if (destDir != null)
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(destDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.EntryType is TarEntryType.RegularFile or TarEntryType.V7RegularFile)
|
||||||
|
{
|
||||||
|
entry.ExtractToFile(destPath, overwrite: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Counts entries in a .tar.zst archive without extracting content.
|
||||||
|
/// Used for verification after archive creation.
|
||||||
|
/// </summary>
|
||||||
|
public static int CountEntries(string archiveFile)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var fileStream = new FileStream(
|
||||||
|
archiveFile,
|
||||||
|
FileMode.Open,
|
||||||
|
FileAccess.Read,
|
||||||
|
FileShare.Read,
|
||||||
|
bufferSize: 81920
|
||||||
|
);
|
||||||
|
|
||||||
|
using var zstdStream = new DecompressionStream(fileStream);
|
||||||
|
using var tarReader = new TarReader(zstdStream);
|
||||||
|
|
||||||
|
var count = 0;
|
||||||
|
while (tarReader.GetNextEntry() is not null)
|
||||||
|
{
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger.Error(ex, "Failed to count entries in archive {File}", archiveFile);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,122 +0,0 @@
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.IO;
|
|
||||||
using System.IO.Compression;
|
|
||||||
using System.Net.Http;
|
|
||||||
using Server.Text;
|
|
||||||
|
|
||||||
namespace Server.Compression
|
|
||||||
{
|
|
||||||
public static class TarArchive
|
|
||||||
{
|
|
||||||
private const string _libArchiveWindowsUrl = @"https://www.libarchive.org/downloads/libarchive-v3.5.2-win64.zip";
|
|
||||||
private static string _pathToTar;
|
|
||||||
|
|
||||||
private static string GetPathToTar()
|
|
||||||
{
|
|
||||||
if (!Core.IsWindows || File.Exists(@"C:\Windows\system32\tar.exe"))
|
|
||||||
{
|
|
||||||
return "tar";
|
|
||||||
}
|
|
||||||
|
|
||||||
var tarPath = Path.Combine(Core.BaseDirectory, "bsdtar/bsdtar.exe");
|
|
||||||
return File.Exists(tarPath) ? tarPath : DownloadTarForWindows();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static int RunTar(string arguments, string compressionProgramPath)
|
|
||||||
{
|
|
||||||
var process = new Process
|
|
||||||
{
|
|
||||||
StartInfo = new ProcessStartInfo
|
|
||||||
{
|
|
||||||
FileName = _pathToTar,
|
|
||||||
Arguments = arguments
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if (compressionProgramPath != null)
|
|
||||||
{
|
|
||||||
var envPaths = $"{process.StartInfo.EnvironmentVariables["PATH"]}:{compressionProgramPath}";
|
|
||||||
process.StartInfo.EnvironmentVariables["PATH"] = envPaths;
|
|
||||||
}
|
|
||||||
|
|
||||||
process.Start();
|
|
||||||
process.WaitForExit();
|
|
||||||
|
|
||||||
return process.ExitCode;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string DownloadTarForWindows()
|
|
||||||
{
|
|
||||||
var tempDir = PathUtility.EnsureRandomPath(Path.GetTempPath());
|
|
||||||
|
|
||||||
var libarchiveFile = Path.Combine(tempDir, "libarchive.zip");
|
|
||||||
// This isn't called often so we don't need to optimize
|
|
||||||
using (var hc = new HttpClient())
|
|
||||||
{
|
|
||||||
var result = hc.Send(new HttpRequestMessage(HttpMethod.Get, new Uri(_libArchiveWindowsUrl)));
|
|
||||||
using var stream = result.Content.ReadAsStream();
|
|
||||||
using var fs = new FileStream(libarchiveFile, FileMode.Create, FileAccess.Write, FileShare.None);
|
|
||||||
stream.CopyTo(fs);
|
|
||||||
}
|
|
||||||
|
|
||||||
ZipFile.ExtractToDirectory(libarchiveFile, tempDir);
|
|
||||||
var libArchivePath = Path.Combine(tempDir, "libarchive");
|
|
||||||
PathUtility.MoveDirectoryContents(Path.Combine(libArchivePath, "bin"), Path.Combine(Core.BaseDirectory, "bsdtar"));
|
|
||||||
File.Delete(libarchiveFile);
|
|
||||||
|
|
||||||
return Path.Combine(Core.BaseDirectory, "bsdtar/bsdtar.exe");
|
|
||||||
}
|
|
||||||
|
|
||||||
public static bool ExtractToDirectory(
|
|
||||||
string fileNamePath,
|
|
||||||
string outputDirectory,
|
|
||||||
string compressCommand = null,
|
|
||||||
string compressionProgramPath = null
|
|
||||||
)
|
|
||||||
{
|
|
||||||
_pathToTar ??= GetPathToTar();
|
|
||||||
|
|
||||||
var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : "";
|
|
||||||
var arguments = $"{useExternalCompression} -xf \"{fileNamePath}\" -C \"{outputDirectory}\"";
|
|
||||||
|
|
||||||
return RunTar(arguments, compressionProgramPath) == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static bool CreateFromPaths(
|
|
||||||
IEnumerable<string> paths,
|
|
||||||
string destinationArchiveFileName,
|
|
||||||
string relativeTo,
|
|
||||||
string compressCommand = null,
|
|
||||||
string compressionProgramPath = null
|
|
||||||
)
|
|
||||||
{
|
|
||||||
_pathToTar ??= GetPathToTar();
|
|
||||||
|
|
||||||
new FileInfo(destinationArchiveFileName).EnsureDirectory();
|
|
||||||
|
|
||||||
using var sb = ValueStringBuilder.Create();
|
|
||||||
var i = 0;
|
|
||||||
foreach (var path in paths)
|
|
||||||
{
|
|
||||||
var relativePath = Path.GetRelativePath(relativeTo, path);
|
|
||||||
if (i++ > 0)
|
|
||||||
{
|
|
||||||
sb.Append($" \"{relativePath}\"");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
sb.Append($"\"{relativePath}\"");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var pathsToCompress = sb.ToString();
|
|
||||||
|
|
||||||
var tarFlags = compressCommand == null ? "-acf" : "-cf";
|
|
||||||
var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : "";
|
|
||||||
var arguments = $"{useExternalCompression}{tarFlags} \"{destinationArchiveFileName}\" -C \"{relativeTo}\" {pathsToCompress}";
|
|
||||||
|
|
||||||
return RunTar(arguments, compressionProgramPath) == 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,95 +0,0 @@
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.IO;
|
|
||||||
using System.Reflection;
|
|
||||||
|
|
||||||
namespace Server.Compression
|
|
||||||
{
|
|
||||||
public static class ZstdArchive
|
|
||||||
{
|
|
||||||
private static readonly string _pathToZstd = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory?.FullName;
|
|
||||||
|
|
||||||
public static bool ExtractToDirectory(string fileNamePath, string outputDirectory)
|
|
||||||
{
|
|
||||||
// bsdtar has a bug and hangs, so we are doing it in two steps.
|
|
||||||
if (Core.IsWindows)
|
|
||||||
{
|
|
||||||
var tempDir = PathUtility.EnsureRandomPath(Path.GetTempPath());
|
|
||||||
var tempTarArchive = Path.Combine(tempDir, "temp-file.tar");
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var process = new Process
|
|
||||||
{
|
|
||||||
StartInfo = new ProcessStartInfo
|
|
||||||
{
|
|
||||||
FileName = Path.Combine(_pathToZstd, "zstd.exe"),
|
|
||||||
Arguments = $"-q -d \"{fileNamePath}\" -o \"{tempTarArchive}\""
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
process.Start();
|
|
||||||
process.WaitForExit();
|
|
||||||
|
|
||||||
return process.ExitCode == 0 && TarArchive.ExtractToDirectory(tempTarArchive, outputDirectory);
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
File.Delete(tempTarArchive);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return TarArchive.ExtractToDirectory(fileNamePath, outputDirectory, "zstd -q -d", _pathToZstd);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static bool CreateFromPaths(
|
|
||||||
IEnumerable<string> paths,
|
|
||||||
string destinationArchiveFileName,
|
|
||||||
string relativeTo
|
|
||||||
)
|
|
||||||
{
|
|
||||||
// bsdtar has a bug and hangs, so we are doing it in two steps.
|
|
||||||
if (Core.IsWindows)
|
|
||||||
{
|
|
||||||
var tempDir = PathUtility.EnsureRandomPath(Path.GetTempPath());
|
|
||||||
var tempTarArchive = Path.Combine(tempDir, "temp-file.tar");
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (!TarArchive.CreateFromPaths(paths, tempTarArchive, relativeTo))
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
var process = new Process
|
|
||||||
{
|
|
||||||
StartInfo = new ProcessStartInfo
|
|
||||||
{
|
|
||||||
FileName = Path.Combine(_pathToZstd, "zstd.exe"),
|
|
||||||
Arguments = $"-q \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\""
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
process.Start();
|
|
||||||
process.WaitForExit();
|
|
||||||
|
|
||||||
return process.ExitCode == 0;
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
Directory.Delete(tempDir, true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return TarArchive.CreateFromPaths(paths, destinationArchiveFileName, relativeTo, "zstd -q", _pathToZstd);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -29,8 +29,7 @@
|
||||||
<Delete Files="..\..\Distribution\Assemblies\LibDeflate.Bindings.dll" ContinueOnError="true" />
|
<Delete Files="..\..\Distribution\Assemblies\LibDeflate.Bindings.dll" ContinueOnError="true" />
|
||||||
<Delete Files="..\..\Distribution\Assemblies\libdeflate.dll" ContinueOnError="true" />
|
<Delete Files="..\..\Distribution\Assemblies\libdeflate.dll" ContinueOnError="true" />
|
||||||
<Delete Files="..\..\Distribution\Assemblies\libdeflate.dylib" ContinueOnError="true" />
|
<Delete Files="..\..\Distribution\Assemblies\libdeflate.dylib" ContinueOnError="true" />
|
||||||
<Delete Files="..\..\Distribution\Assemblies\Zstd.Binaries.dll" ContinueOnError="true" />
|
<Delete Files="..\..\Distribution\Assemblies\ZstdNet.dll" ContinueOnError="true" />
|
||||||
<Delete Files="..\..\Distribution\Assemblies\zstd.exe" ContinueOnError="true" />
|
|
||||||
<Delete Files="..\..\Distribution\Assemblies\ModernUO.Serialization.Annotations.dll" ContinueOnError="true" />
|
<Delete Files="..\..\Distribution\Assemblies\ModernUO.Serialization.Annotations.dll" ContinueOnError="true" />
|
||||||
</Target>
|
</Target>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|
@ -45,7 +44,7 @@
|
||||||
<PackageReference Include="Argon2.Bindings" Version="1.17.0" />
|
<PackageReference Include="Argon2.Bindings" Version="1.17.0" />
|
||||||
<PackageReference Include="ModernUO.CodeGeneratedEvents.Annotations" Version="1.0.0" />
|
<PackageReference Include="ModernUO.CodeGeneratedEvents.Annotations" Version="1.0.0" />
|
||||||
<PackageReference Include="ModernUO.CodeGeneratedEvents.Generator" Version="1.0.3.2" PrivateAssets="all" />
|
<PackageReference Include="ModernUO.CodeGeneratedEvents.Generator" Version="1.0.3.2" PrivateAssets="all" />
|
||||||
<PackageReference Include="Zstd.Binaries" Version="1.6.0" />
|
<PackageReference Include="ZstdNet" Version="1.5.7" />
|
||||||
|
|
||||||
<PackageReference Include="ModernUO.Serialization.Annotations" Version="2.14.2" />
|
<PackageReference Include="ModernUO.Serialization.Annotations" Version="2.14.2" />
|
||||||
<PackageReference Include="ModernUO.Serialization.Generator" Version="2.14.2" />
|
<PackageReference Include="ModernUO.Serialization.Generator" Version="2.14.2" />
|
||||||
|
|
|
||||||
34
Projects/UOContent/World Saves/ArchiveDestinationRegistry.cs
Normal file
34
Projects/UOContent/World Saves/ArchiveDestinationRegistry.cs
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
/*************************************************************************
|
||||||
|
* ModernUO *
|
||||||
|
* Copyright 2019-2026 - ModernUO Development Team *
|
||||||
|
* Email: hi@modernuo.com *
|
||||||
|
* File: ArchiveDestinationRegistry.cs *
|
||||||
|
* *
|
||||||
|
* This program is free software: you can redistribute it and/or modify *
|
||||||
|
* it under the terms of the GNU General Public License as published by *
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or *
|
||||||
|
* (at your option) any later version. *
|
||||||
|
* *
|
||||||
|
* You should have received a copy of the GNU General Public License *
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||||
|
*************************************************************************/
|
||||||
|
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
namespace Server.Saves;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Registration point for archive destinations. Register during Configure() phase.
|
||||||
|
/// No locks needed — registration is single-threaded, reading happens on one
|
||||||
|
/// ThreadPool thread at a time (guarded by the archive concurrency flag).
|
||||||
|
/// </summary>
|
||||||
|
public static class ArchiveDestinationRegistry
|
||||||
|
{
|
||||||
|
private static readonly List<IArchiveDestination> _destinations = [];
|
||||||
|
|
||||||
|
public static IReadOnlyList<IArchiveDestination> Destinations => _destinations;
|
||||||
|
|
||||||
|
public static void Register(IArchiveDestination destination) => _destinations.Add(destination);
|
||||||
|
|
||||||
|
public static void Unregister(IArchiveDestination destination) => _destinations.Remove(destination);
|
||||||
|
}
|
||||||
23
Projects/UOContent/World Saves/ArchiveEnums.cs
Normal file
23
Projects/UOContent/World Saves/ArchiveEnums.cs
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
/*************************************************************************
|
||||||
|
* ModernUO *
|
||||||
|
* Copyright 2019-2026 - ModernUO Development Team *
|
||||||
|
* Email: hi@modernuo.com *
|
||||||
|
* File: ArchiveEnums.cs *
|
||||||
|
* *
|
||||||
|
* This program is free software: you can redistribute it and/or modify *
|
||||||
|
* it under the terms of the GNU General Public License as published by *
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or *
|
||||||
|
* (at your option) any later version. *
|
||||||
|
* *
|
||||||
|
* You should have received a copy of the GNU General Public License *
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||||
|
*************************************************************************/
|
||||||
|
|
||||||
|
namespace Server.Saves;
|
||||||
|
|
||||||
|
public enum ArchivePeriod
|
||||||
|
{
|
||||||
|
Hourly,
|
||||||
|
Daily,
|
||||||
|
Monthly
|
||||||
|
}
|
||||||
40
Projects/UOContent/World Saves/ArchiveEventArgs.cs
Normal file
40
Projects/UOContent/World Saves/ArchiveEventArgs.cs
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
/*************************************************************************
|
||||||
|
* ModernUO *
|
||||||
|
* Copyright 2019-2026 - ModernUO Development Team *
|
||||||
|
* Email: hi@modernuo.com *
|
||||||
|
* File: ArchiveEventArgs.cs *
|
||||||
|
* *
|
||||||
|
* This program is free software: you can redistribute it and/or modify *
|
||||||
|
* it under the terms of the GNU General Public License as published by *
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or *
|
||||||
|
* (at your option) any later version. *
|
||||||
|
* *
|
||||||
|
* You should have received a copy of the GNU General Public License *
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||||
|
*************************************************************************/
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace Server.Saves;
|
||||||
|
|
||||||
|
public class ArchiveCompletedEventArgs(
|
||||||
|
string archiveFilePath,
|
||||||
|
ArchivePeriod period,
|
||||||
|
DateTime rangeStart,
|
||||||
|
long fileSizeBytes,
|
||||||
|
double elapsedSeconds
|
||||||
|
)
|
||||||
|
{
|
||||||
|
public string ArchiveFilePath { get; } = archiveFilePath;
|
||||||
|
public ArchivePeriod Period { get; } = period;
|
||||||
|
public DateTime RangeStart { get; } = rangeStart;
|
||||||
|
public long FileSizeBytes { get; } = fileSizeBytes;
|
||||||
|
public double ElapsedSeconds { get; } = elapsedSeconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ArchiveFailedEventArgs(ArchivePeriod period, DateTime rangeStart, Exception exception)
|
||||||
|
{
|
||||||
|
public ArchivePeriod Period { get; } = period;
|
||||||
|
public DateTime RangeStart { get; } = rangeStart;
|
||||||
|
public Exception Exception { get; } = exception;
|
||||||
|
}
|
||||||
269
Projects/UOContent/World Saves/ArchiveJournal.cs
Normal file
269
Projects/UOContent/World Saves/ArchiveJournal.cs
Normal file
|
|
@ -0,0 +1,269 @@
|
||||||
|
/*************************************************************************
|
||||||
|
* ModernUO *
|
||||||
|
* Copyright 2019-2026 - ModernUO Development Team *
|
||||||
|
* Email: hi@modernuo.com *
|
||||||
|
* File: ArchiveJournal.cs *
|
||||||
|
* *
|
||||||
|
* This program is free software: you can redistribute it and/or modify *
|
||||||
|
* it under the terms of the GNU General Public License as published by *
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or *
|
||||||
|
* (at your option) any later version. *
|
||||||
|
* *
|
||||||
|
* You should have received a copy of the GNU General Public License *
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||||
|
*************************************************************************/
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using Server.Json;
|
||||||
|
using Server.Logging;
|
||||||
|
|
||||||
|
namespace Server.Saves;
|
||||||
|
|
||||||
|
public enum ArchiveOperationState
|
||||||
|
{
|
||||||
|
Started,
|
||||||
|
Archived,
|
||||||
|
Distributed,
|
||||||
|
Completed,
|
||||||
|
Failed
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ArchiveJournalEntry
|
||||||
|
{
|
||||||
|
public string Id { get; set; }
|
||||||
|
public ArchivePeriod Period { get; set; }
|
||||||
|
public DateTime RangeStart { get; set; }
|
||||||
|
public ArchiveOperationState State { get; set; }
|
||||||
|
public string ArchiveFile { get; set; }
|
||||||
|
public string TempFile { get; set; }
|
||||||
|
public List<string> SourceDirectories { get; set; } = [];
|
||||||
|
public int EntryCount { get; set; }
|
||||||
|
public long TotalSourceBytes { get; set; }
|
||||||
|
public long ArchiveBytes { get; set; }
|
||||||
|
public DateTime StartedAt { get; set; }
|
||||||
|
public DateTime CompletedAt { get; set; } = DateTime.MaxValue;
|
||||||
|
public string FailureReason { get; set; }
|
||||||
|
public Dictionary<string, bool> DestinationResults { get; set; } = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ArchiveJournalData
|
||||||
|
{
|
||||||
|
public List<ArchiveJournalEntry> Operations { get; set; } = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class ArchiveJournal
|
||||||
|
{
|
||||||
|
private const int MaxJournalEntries = 100;
|
||||||
|
|
||||||
|
private static readonly ILogger logger = LogFactory.GetLogger(typeof(ArchiveJournal));
|
||||||
|
|
||||||
|
private static string _journalPath;
|
||||||
|
private static ArchiveJournalData _data = new();
|
||||||
|
|
||||||
|
public static IReadOnlyList<ArchiveJournalEntry> Operations => _data.Operations;
|
||||||
|
|
||||||
|
public static void Configure(string archivePath)
|
||||||
|
{
|
||||||
|
_journalPath = Path.Combine(archivePath, ".archive-journal.json");
|
||||||
|
_data = new ArchiveJournalData();
|
||||||
|
Load();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void Load()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (File.Exists(_journalPath))
|
||||||
|
{
|
||||||
|
_data = JsonConfig.Deserialize<ArchiveJournalData>(_journalPath) ?? new ArchiveJournalData();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger.Error(ex, "Failed to load archive journal, starting fresh");
|
||||||
|
_data = new ArchiveJournalData();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void Save()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var tempPath = _journalPath + ".tmp";
|
||||||
|
JsonConfig.Serialize(tempPath, _data);
|
||||||
|
|
||||||
|
if (File.Exists(_journalPath))
|
||||||
|
{
|
||||||
|
File.Delete(_journalPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
File.Move(tempPath, _journalPath);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger.Error(ex, "Failed to save archive journal");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void Prune()
|
||||||
|
{
|
||||||
|
if (_data.Operations.Count > MaxJournalEntries)
|
||||||
|
{
|
||||||
|
// Remove oldest completed/failed entries beyond the limit
|
||||||
|
var toRemove = _data.Operations.Count - MaxJournalEntries;
|
||||||
|
for (var i = _data.Operations.Count - 1; i >= 0 && toRemove > 0; i--)
|
||||||
|
{
|
||||||
|
if (_data.Operations[i].State is ArchiveOperationState.Completed or ArchiveOperationState.Failed)
|
||||||
|
{
|
||||||
|
_data.Operations.RemoveAt(i);
|
||||||
|
toRemove--;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ArchiveJournalEntry BeginOperation(
|
||||||
|
ArchivePeriod period,
|
||||||
|
DateTime rangeStart,
|
||||||
|
string tempFile,
|
||||||
|
string archiveFile,
|
||||||
|
List<string> sourceDirectories)
|
||||||
|
{
|
||||||
|
var entry = new ArchiveJournalEntry
|
||||||
|
{
|
||||||
|
Id = $"{rangeStart:O}-{period.ToString().ToLowerInvariant()}",
|
||||||
|
Period = period,
|
||||||
|
RangeStart = rangeStart,
|
||||||
|
State = ArchiveOperationState.Started,
|
||||||
|
TempFile = tempFile,
|
||||||
|
ArchiveFile = archiveFile,
|
||||||
|
SourceDirectories = sourceDirectories,
|
||||||
|
StartedAt = Core.Now
|
||||||
|
};
|
||||||
|
|
||||||
|
_data.Operations.Add(entry);
|
||||||
|
Prune();
|
||||||
|
Save();
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void RecordArchived(ArchiveJournalEntry entry, int entryCount, long archiveBytes)
|
||||||
|
{
|
||||||
|
entry.State = ArchiveOperationState.Archived;
|
||||||
|
entry.EntryCount = entryCount;
|
||||||
|
entry.ArchiveBytes = archiveBytes;
|
||||||
|
entry.TempFile = null; // Temp file has been renamed to final
|
||||||
|
Save();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void RecordDistributed(ArchiveJournalEntry entry, Dictionary<string, bool> results)
|
||||||
|
{
|
||||||
|
entry.State = ArchiveOperationState.Distributed;
|
||||||
|
entry.DestinationResults = results;
|
||||||
|
Save();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void RecordCompleted(ArchiveJournalEntry entry)
|
||||||
|
{
|
||||||
|
entry.State = ArchiveOperationState.Completed;
|
||||||
|
entry.CompletedAt = Core.Now;
|
||||||
|
Save();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void RecordFailure(ArchiveJournalEntry entry, string reason)
|
||||||
|
{
|
||||||
|
entry.State = ArchiveOperationState.Failed;
|
||||||
|
entry.FailureReason = reason;
|
||||||
|
entry.CompletedAt = Core.Now;
|
||||||
|
Save();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Recovers from interrupted operations on startup.
|
||||||
|
/// Handles missing files/directories gracefully (e.g. admin cleaned up archives folder).
|
||||||
|
/// </summary>
|
||||||
|
public static void RecoverInterrupted()
|
||||||
|
{
|
||||||
|
var recovered = false;
|
||||||
|
|
||||||
|
foreach (var entry in _data.Operations)
|
||||||
|
{
|
||||||
|
switch (entry.State)
|
||||||
|
{
|
||||||
|
case ArchiveOperationState.Started:
|
||||||
|
{
|
||||||
|
// Archive creation was interrupted. Clean up temp file if it exists.
|
||||||
|
if (entry.TempFile != null)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (File.Exists(entry.TempFile))
|
||||||
|
{
|
||||||
|
File.Delete(entry.TempFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logger.Warning(ex, "Failed to clean up temp file {File}", entry.TempFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Warning(
|
||||||
|
"Recovered interrupted archive operation {Id} (was in Started state)",
|
||||||
|
entry.Id
|
||||||
|
);
|
||||||
|
entry.State = ArchiveOperationState.Failed;
|
||||||
|
entry.FailureReason = "Interrupted during archive creation";
|
||||||
|
entry.CompletedAt = Core.Now;
|
||||||
|
recovered = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case ArchiveOperationState.Archived:
|
||||||
|
{
|
||||||
|
// Archive was created but not distributed.
|
||||||
|
// If the archive file was deleted (admin cleanup), mark as failed.
|
||||||
|
if (entry.ArchiveFile != null && !File.Exists(entry.ArchiveFile))
|
||||||
|
{
|
||||||
|
logger.Warning(
|
||||||
|
"Archive file for operation {Id} no longer exists (manually deleted?). Marking failed.",
|
||||||
|
entry.Id
|
||||||
|
);
|
||||||
|
entry.State = ArchiveOperationState.Failed;
|
||||||
|
entry.FailureReason = "Archive file missing on recovery";
|
||||||
|
entry.CompletedAt = Core.Now;
|
||||||
|
recovered = true;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.Warning(
|
||||||
|
"Found unfinished archive operation {Id} (Archived but not distributed). " +
|
||||||
|
"Will re-attempt distribution on next cycle.",
|
||||||
|
entry.Id
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case ArchiveOperationState.Distributed:
|
||||||
|
{
|
||||||
|
// Sources should have been pruned but weren't.
|
||||||
|
logger.Information(
|
||||||
|
"Completing interrupted operation {Id} (was Distributed, pruning sources)",
|
||||||
|
entry.Id
|
||||||
|
);
|
||||||
|
entry.State = ArchiveOperationState.Completed;
|
||||||
|
entry.CompletedAt = Core.Now;
|
||||||
|
recovered = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (recovered)
|
||||||
|
{
|
||||||
|
Save();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load diff
46
Projects/UOContent/World Saves/IArchiveDestination.cs
Normal file
46
Projects/UOContent/World Saves/IArchiveDestination.cs
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
/*************************************************************************
|
||||||
|
* ModernUO *
|
||||||
|
* Copyright 2019-2026 - ModernUO Development Team *
|
||||||
|
* Email: hi@modernuo.com *
|
||||||
|
* File: IArchiveDestination.cs *
|
||||||
|
* *
|
||||||
|
* This program is free software: you can redistribute it and/or modify *
|
||||||
|
* it under the terms of the GNU General Public License as published by *
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or *
|
||||||
|
* (at your option) any later version. *
|
||||||
|
* *
|
||||||
|
* You should have received a copy of the GNU General Public License *
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||||
|
*************************************************************************/
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace Server.Saves;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Represents a destination for completed archive files.
|
||||||
|
/// Implementations run on a background thread (ThreadPool).
|
||||||
|
/// </summary>
|
||||||
|
public interface IArchiveDestination
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// A human-readable name for logging (e.g., "Local Filesystem", "S3 us-east-1").
|
||||||
|
/// </summary>
|
||||||
|
string Name { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Called after a local archive file is successfully created and verified.
|
||||||
|
/// Must be safe to call from a ThreadPool thread.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="archiveFilePath">Full path to the verified .tar.zst archive.</param>
|
||||||
|
/// <param name="period">The archive period (Hourly, Daily, Monthly).</param>
|
||||||
|
/// <param name="rangeStart">The start of the time range this archive covers.</param>
|
||||||
|
/// <returns>True if the destination successfully received the archive.</returns>
|
||||||
|
bool SendArchive(string archiveFilePath, ArchivePeriod period, DateTime rangeStart);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns the number of local archives to retain for this period.
|
||||||
|
/// Destinations that keep their own copies may return 0.
|
||||||
|
/// </summary>
|
||||||
|
int GetRetentionCount(ArchivePeriod period);
|
||||||
|
}
|
||||||
50
Projects/UOContent/World Saves/LocalArchiveDestination.cs
Normal file
50
Projects/UOContent/World Saves/LocalArchiveDestination.cs
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
/*************************************************************************
|
||||||
|
* ModernUO *
|
||||||
|
* Copyright 2019-2026 - ModernUO Development Team *
|
||||||
|
* Email: hi@modernuo.com *
|
||||||
|
* File: LocalArchiveDestination.cs *
|
||||||
|
* *
|
||||||
|
* This program is free software: you can redistribute it and/or modify *
|
||||||
|
* it under the terms of the GNU General Public License as published by *
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or *
|
||||||
|
* (at your option) any later version. *
|
||||||
|
* *
|
||||||
|
* You should have received a copy of the GNU General Public License *
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||||
|
*************************************************************************/
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace Server.Saves;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Built-in archive destination representing the local filesystem.
|
||||||
|
/// The archive is already on local disk, so SendArchive is a no-op.
|
||||||
|
/// Participates in the retention/pruning protocol via configurable counts.
|
||||||
|
/// </summary>
|
||||||
|
public class LocalArchiveDestination : IArchiveDestination
|
||||||
|
{
|
||||||
|
public string Name => "Local Filesystem";
|
||||||
|
|
||||||
|
private readonly int _hourlyRetention;
|
||||||
|
private readonly int _dailyRetention;
|
||||||
|
private readonly int _monthlyRetention;
|
||||||
|
|
||||||
|
public LocalArchiveDestination(int hourlyRetention, int dailyRetention, int monthlyRetention)
|
||||||
|
{
|
||||||
|
_hourlyRetention = hourlyRetention;
|
||||||
|
_dailyRetention = dailyRetention;
|
||||||
|
_monthlyRetention = monthlyRetention;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Archive is already on local filesystem — nothing to do.
|
||||||
|
public bool SendArchive(string archiveFilePath, ArchivePeriod period, DateTime rangeStart) => true;
|
||||||
|
|
||||||
|
public int GetRetentionCount(ArchivePeriod period) => period switch
|
||||||
|
{
|
||||||
|
ArchivePeriod.Hourly => _hourlyRetention,
|
||||||
|
ArchivePeriod.Daily => _dailyRetention,
|
||||||
|
ArchivePeriod.Monthly => _monthlyRetention,
|
||||||
|
_ => 0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -1,68 +1,104 @@
|
||||||
using Server.Saves;
|
using Server.Saves;
|
||||||
|
|
||||||
namespace Server.Commands
|
namespace Server.Commands;
|
||||||
|
|
||||||
|
public static class SaveCommands
|
||||||
{
|
{
|
||||||
public static class SaveCommands
|
public static void Configure()
|
||||||
{
|
{
|
||||||
public static void Configure()
|
CommandSystem.Register("Save", AccessLevel.Administrator, Save_OnCommand);
|
||||||
|
CommandSystem.Register("SetSaves", AccessLevel.Administrator, SetAutoSaves_OnCommand);
|
||||||
|
CommandSystem.Register("AutoSave", AccessLevel.Administrator, SetAutoSaves_OnCommand);
|
||||||
|
CommandSystem.Register("SaveFrequency", AccessLevel.Administrator, SetSaveFrequency_OnCommand);
|
||||||
|
CommandSystem.Register("PruneArchives", AccessLevel.Administrator, PruneArchives_OnCommand);
|
||||||
|
CommandSystem.Register("ArchiveStatus", AccessLevel.Administrator, ArchiveStatus_OnCommand);
|
||||||
|
CommandSystem.Register("ArchiveNow", AccessLevel.Administrator, ArchiveNow_OnCommand);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Usage("PruneArchives"), Description("Prunes archives folder.")]
|
||||||
|
private static void PruneArchives_OnCommand(CommandEventArgs e) => AutoArchive.PruneBackups();
|
||||||
|
|
||||||
|
[Usage("Save"), Description("Saves the world.")]
|
||||||
|
private static void Save_OnCommand(CommandEventArgs e) => AutoSave.Save();
|
||||||
|
|
||||||
|
[Usage("AutoSave <on | off>"), Description("Enables or disables automatic shard saving.")]
|
||||||
|
public static void SetAutoSaves_OnCommand(CommandEventArgs e)
|
||||||
|
{
|
||||||
|
if (e.Length == 1)
|
||||||
{
|
{
|
||||||
CommandSystem.Register("Save", AccessLevel.Administrator, Save_OnCommand);
|
var enabled = AutoSave.SavesEnabled = e.GetBoolean(0);
|
||||||
CommandSystem.Register("SetSaves", AccessLevel.Administrator, SetAutoSaves_OnCommand);
|
|
||||||
CommandSystem.Register("AutoSave", AccessLevel.Administrator, SetAutoSaves_OnCommand);
|
e.Mobile.SendMessage(enabled ? "Saves have been enabled." : "Saves have been disabled.");
|
||||||
CommandSystem.Register("SaveFrequency", AccessLevel.Administrator, SetSaveFrequency_OnCommand);
|
|
||||||
CommandSystem.Register("PruneArchives", AccessLevel.Administrator, PruneArchives_OnCommand);
|
|
||||||
}
|
}
|
||||||
|
else
|
||||||
[Usage("PruneArchives"), Description("Prunes archives folder.")]
|
|
||||||
private static void PruneArchives_OnCommand(CommandEventArgs e)
|
|
||||||
{
|
{
|
||||||
AutoArchive.PruneBackups();
|
e.Mobile.SendMessage("Format: AutoSave <on | off>");
|
||||||
}
|
|
||||||
|
|
||||||
[Usage("Save"), Description("Saves the world.")]
|
|
||||||
private static void Save_OnCommand(CommandEventArgs e)
|
|
||||||
{
|
|
||||||
AutoSave.Save();
|
|
||||||
}
|
|
||||||
|
|
||||||
[Usage("AutoSave <on | off>"), Description("Enables or disables automatic shard saving.")]
|
|
||||||
public static void SetAutoSaves_OnCommand(CommandEventArgs e)
|
|
||||||
{
|
|
||||||
if (e.Length == 1)
|
|
||||||
{
|
|
||||||
|
|
||||||
var enabled = AutoSave.SavesEnabled = e.GetBoolean(0);
|
|
||||||
|
|
||||||
if (enabled)
|
|
||||||
{
|
|
||||||
e.Mobile.SendMessage("Saves have been enabled.");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
e.Mobile.SendMessage("Saves have been disabled.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
e.Mobile.SendMessage("Format: AutoSave <on | off>");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Usage("SaveFrequency <duration> [warning duration]"), Description("Sets the save frequency starting at midnight local to the server.")]
|
|
||||||
public static void SetSaveFrequency_OnCommand(CommandEventArgs e)
|
|
||||||
{
|
|
||||||
if (e.Length < 1)
|
|
||||||
{
|
|
||||||
e.Mobile.SendMessage("Format: SaveFrequency <duration> [warning duration]");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var saveDelay = e.GetTimeSpan(0);
|
|
||||||
var warningDelay = e.Length >= 2 ? e.GetTimeSpan(1) : AutoSave.Warning;
|
|
||||||
|
|
||||||
AutoSave.ResetAutoSave(saveDelay, warningDelay);
|
|
||||||
|
|
||||||
e.Mobile.SendMessage("Save frequency has been updated.");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Usage("SaveFrequency <duration> [warning duration]"), Description("Sets the save frequency starting at midnight local to the server.")]
|
||||||
|
public static void SetSaveFrequency_OnCommand(CommandEventArgs e)
|
||||||
|
{
|
||||||
|
if (e.Length < 1)
|
||||||
|
{
|
||||||
|
e.Mobile.SendMessage("Format: SaveFrequency <duration> [warning duration]");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var saveDelay = e.GetTimeSpan(0);
|
||||||
|
var warningDelay = e.Length >= 2 ? e.GetTimeSpan(1) : AutoSave.Warning;
|
||||||
|
|
||||||
|
AutoSave.ResetAutoSave(saveDelay, warningDelay);
|
||||||
|
|
||||||
|
e.Mobile.SendMessage("Save frequency has been updated.");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Usage("ArchiveStatus")]
|
||||||
|
[Description("Shows archive system status, journal state, and registered destinations.")]
|
||||||
|
private static void ArchiveStatus_OnCommand(CommandEventArgs e)
|
||||||
|
{
|
||||||
|
var mobile = e.Mobile;
|
||||||
|
|
||||||
|
mobile.SendMessage("--- Archive Status ---");
|
||||||
|
mobile.SendMessage($"Next hourly: {AutoArchive.NextHourlyArchive:yyyy-MM-dd HH:mm}");
|
||||||
|
mobile.SendMessage($"Next daily: {AutoArchive.NextDailyArchive:yyyy-MM-dd}");
|
||||||
|
mobile.SendMessage($"Next monthly: {AutoArchive.NextMonthlyArchive:yyyy-MM}");
|
||||||
|
|
||||||
|
var destinations = ArchiveDestinationRegistry.Destinations;
|
||||||
|
mobile.SendMessage($"Registered destinations: {destinations.Count}");
|
||||||
|
foreach (var dest in destinations)
|
||||||
|
{
|
||||||
|
mobile.SendMessage(
|
||||||
|
$" - {dest.Name} (retention: {dest.GetRetentionCount(ArchivePeriod.Hourly)}h/" +
|
||||||
|
$"{dest.GetRetentionCount(ArchivePeriod.Daily)}d/" +
|
||||||
|
$"{dest.GetRetentionCount(ArchivePeriod.Monthly)}m)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
var operations = ArchiveJournal.Operations;
|
||||||
|
var pending = 0;
|
||||||
|
var failed = 0;
|
||||||
|
for (var i = 0; i < operations.Count; i++)
|
||||||
|
{
|
||||||
|
var op = operations[i];
|
||||||
|
if (op.State is ArchiveOperationState.Started or ArchiveOperationState.Archived
|
||||||
|
or ArchiveOperationState.Distributed)
|
||||||
|
{
|
||||||
|
pending++;
|
||||||
|
}
|
||||||
|
else if (op.State == ArchiveOperationState.Failed)
|
||||||
|
{
|
||||||
|
failed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
mobile.SendMessage($"Journal entries: {operations.Count} ({pending} pending, {failed} failed)");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Usage("ArchiveNow"), Description("Forces an immediate archive rollup regardless of schedule.")]
|
||||||
|
private static void ArchiveNow_OnCommand(CommandEventArgs e)
|
||||||
|
{
|
||||||
|
e.Mobile.SendMessage("Forcing immediate archive rollup...");
|
||||||
|
AutoArchive.ForceRollup();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue