diff --git a/Projects/UOContent.Tests/Tests/WorldSaves/ArchiveJournalTests.cs b/Projects/UOContent.Tests/Tests/WorldSaves/ArchiveJournalTests.cs new file mode 100644 index 000000000..ede98a27f --- /dev/null +++ b/Projects/UOContent.Tests/Tests/WorldSaves/ArchiveJournalTests.cs @@ -0,0 +1,230 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Server.Saves; +using Xunit; + +namespace Server.Tests; + +[Collection("Sequential UOContent Tests")] +public class ArchiveJournalTests : IDisposable +{ + private readonly string _testDir; + + public ArchiveJournalTests() + { + _testDir = Path.Combine(Path.GetTempPath(), $"modernuo-journal-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_testDir); + } + + public void Dispose() + { + if (Directory.Exists(_testDir)) + { + Directory.Delete(_testDir, true); + } + } + + [Fact] + public void Configure_LoadsCleanState() + { + // Arrange & Act + ArchiveJournal.Configure(_testDir); + + // Assert - fresh directory should have no operations + // Note: ArchiveJournal is static, so we check the journal file doesn't exist + var journalPath = Path.Combine(_testDir, ".archive-journal.json"); + Assert.False(File.Exists(journalPath)); + } + + [Fact] + public void BeginOperation_CreatesStartedEntry() + { + // Arrange + ArchiveJournal.Configure(_testDir); + + // Act + var entry = ArchiveJournal.BeginOperation( + ArchivePeriod.Hourly, + new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc), + "/tmp/archive.tar.zst.tmp", + "/archives/hourly/2026-03-22-14.tar.zst", + ["backup1", "backup2"] + ); + + // Assert + Assert.Equal(ArchiveOperationState.Started, entry.State); + Assert.Equal(ArchivePeriod.Hourly, entry.Period); + Assert.Equal("/tmp/archive.tar.zst.tmp", entry.TempFile); + Assert.Equal(2, entry.SourceDirectories.Count); + Assert.Contains(entry, ArchiveJournal.Operations); + } + + [Fact] + public void RecordArchived_TransitionsState() + { + // Arrange + ArchiveJournal.Configure(_testDir); + var entry = ArchiveJournal.BeginOperation( + ArchivePeriod.Daily, + new DateTime(2026, 3, 22, 0, 0, 0, DateTimeKind.Utc), + "/tmp/test.tmp", + "/archives/daily/2026-03-22.tar.zst", + ["backup1"] + ); + + // Act + ArchiveJournal.RecordArchived(entry, 42, 1024000); + + // Assert + Assert.Equal(ArchiveOperationState.Archived, entry.State); + Assert.Equal(42, entry.EntryCount); + Assert.Equal(1024000, entry.ArchiveBytes); + Assert.Null(entry.TempFile); // Temp file cleared after rename + } + + [Fact] + public void RecordDistributed_StoresResults() + { + // Arrange + ArchiveJournal.Configure(_testDir); + var entry = ArchiveJournal.BeginOperation( + ArchivePeriod.Monthly, + new DateTime(2026, 3, 1, 0, 0, 0, DateTimeKind.Utc), + "/tmp/test.tmp", + "/archives/monthly/2026-03.tar.zst", + ["backup1"] + ); + ArchiveJournal.RecordArchived(entry, 100, 5000000); + + var results = new Dictionary + { + { "Local Filesystem", true }, + { "S3 us-east-1", false } + }; + + // Act + ArchiveJournal.RecordDistributed(entry, results); + + // Assert + Assert.Equal(ArchiveOperationState.Distributed, entry.State); + Assert.True(entry.DestinationResults["Local Filesystem"]); + Assert.False(entry.DestinationResults["S3 us-east-1"]); + } + + [Fact] + public void RecordCompleted_IsTerminal() + { + // Arrange + ArchiveJournal.Configure(_testDir); + var entry = ArchiveJournal.BeginOperation( + ArchivePeriod.Hourly, + new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc), + "/tmp/test.tmp", + "/archives/hourly/test.tar.zst", + ["backup1"] + ); + ArchiveJournal.RecordArchived(entry, 10, 500); + ArchiveJournal.RecordDistributed(entry, new Dictionary()); + + // Act + ArchiveJournal.RecordCompleted(entry); + + // Assert + Assert.Equal(ArchiveOperationState.Completed, entry.State); + Assert.NotEqual(DateTime.MaxValue, entry.CompletedAt); + } + + [Fact] + public void RecordFailure_RecordsReason() + { + // Arrange + ArchiveJournal.Configure(_testDir); + var entry = ArchiveJournal.BeginOperation( + ArchivePeriod.Hourly, + new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc), + "/tmp/test.tmp", + "/archives/hourly/test.tar.zst", + ["backup1"] + ); + + // Act + ArchiveJournal.RecordFailure(entry, "Disk full"); + + // Assert + Assert.Equal(ArchiveOperationState.Failed, entry.State); + Assert.Equal("Disk full", entry.FailureReason); + Assert.NotEqual(DateTime.MaxValue, entry.CompletedAt); + } + + [Fact] + public void Journal_PersistsToJsonFile() + { + // Arrange + ArchiveJournal.Configure(_testDir); + ArchiveJournal.BeginOperation( + ArchivePeriod.Hourly, + new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc), + "/tmp/test.tmp", + "/archives/hourly/test.tar.zst", + ["backup1"] + ); + + // Assert - journal file should exist + var journalPath = Path.Combine(_testDir, ".archive-journal.json"); + Assert.True(File.Exists(journalPath)); + + var content = File.ReadAllText(journalPath); + Assert.Contains("Hourly", content); + Assert.Contains("Started", content); + } + + [Fact] + public void RecoverInterrupted_CleansUpStartedOperations() + { + // Arrange + ArchiveJournal.Configure(_testDir); + + // Create a temp file that simulates an interrupted archive + var tempFile = Path.Combine(_testDir, "interrupted.tar.zst.tmp"); + File.WriteAllText(tempFile, "partial archive data"); + + var entry = ArchiveJournal.BeginOperation( + ArchivePeriod.Hourly, + new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc), + tempFile, + Path.Combine(_testDir, "final.tar.zst"), + ["backup1"] + ); + + // Act + ArchiveJournal.RecoverInterrupted(); + + // Assert + Assert.Equal(ArchiveOperationState.Failed, entry.State); + Assert.Equal("Interrupted during archive creation", entry.FailureReason); + Assert.False(File.Exists(tempFile)); // Temp file should be cleaned up + } + + [Fact] + public void RecoverInterrupted_CompletesDistributedOperations() + { + // Arrange + ArchiveJournal.Configure(_testDir); + var entry = ArchiveJournal.BeginOperation( + ArchivePeriod.Daily, + new DateTime(2026, 3, 22, 0, 0, 0, DateTimeKind.Utc), + "/tmp/test.tmp", + "/archives/daily/test.tar.zst", + ["backup1"] + ); + ArchiveJournal.RecordArchived(entry, 50, 2000); + ArchiveJournal.RecordDistributed(entry, new Dictionary { { "Local", true } }); + + // Act - simulate restart + ArchiveJournal.RecoverInterrupted(); + + // Assert - should be marked completed + Assert.Equal(ArchiveOperationState.Completed, entry.State); + } +} diff --git a/Projects/UOContent.Tests/Tests/WorldSaves/AutoArchiveHelperTests.cs b/Projects/UOContent.Tests/Tests/WorldSaves/AutoArchiveHelperTests.cs new file mode 100644 index 000000000..b0d2b99d4 --- /dev/null +++ b/Projects/UOContent.Tests/Tests/WorldSaves/AutoArchiveHelperTests.cs @@ -0,0 +1,104 @@ +using System; +using Server.Saves; +using Xunit; + +namespace Server.Tests; + +[Collection("Sequential UOContent Tests")] +public class AutoArchiveHelperTests +{ + [Fact] + public void ArchiveDestinationRegistry_RegisterAndRetrieve() + { + // Arrange + var dest = new LocalArchiveDestination(24, 30, 12); + + // Act + ArchiveDestinationRegistry.Register(dest); + + // Assert + Assert.Contains(dest, ArchiveDestinationRegistry.Destinations); + + // Cleanup + ArchiveDestinationRegistry.Unregister(dest); + Assert.DoesNotContain(dest, ArchiveDestinationRegistry.Destinations); + } + + [Fact] + public void LocalArchiveDestination_SendArchive_ReturnsTrue() + { + // Arrange + var dest = new LocalArchiveDestination(24, 30, 12); + + // Act + var result = dest.SendArchive("/fake/path.tar.zst", ArchivePeriod.Hourly, DateTime.UtcNow); + + // Assert + Assert.True(result); + } + + [Theory] + [InlineData(ArchivePeriod.Hourly, 24)] + [InlineData(ArchivePeriod.Daily, 30)] + [InlineData(ArchivePeriod.Monthly, 12)] + public void LocalArchiveDestination_RetentionCounts(ArchivePeriod period, int expected) + { + // Arrange + var dest = new LocalArchiveDestination(24, 30, 12); + + // Act & Assert + Assert.Equal(expected, dest.GetRetentionCount(period)); + } + + [Fact] + public void ArchiveCompletedEventArgs_StoresValues() + { + // Arrange & Act + var args = new ArchiveCompletedEventArgs( + "/archives/hourly/test.tar.zst", + ArchivePeriod.Hourly, + new DateTime(2026, 3, 22, 14, 0, 0, DateTimeKind.Utc), + 1048576, + 2.5 + ); + + // Assert + Assert.Equal("/archives/hourly/test.tar.zst", args.ArchiveFilePath); + Assert.Equal(ArchivePeriod.Hourly, args.Period); + Assert.Equal(1048576, args.FileSizeBytes); + Assert.Equal(2.5, args.ElapsedSeconds); + } + + [Fact] + public void ArchiveFailedEventArgs_StoresException() + { + // Arrange + var ex = new InvalidOperationException("disk full"); + + // Act + var args = new ArchiveFailedEventArgs( + ArchivePeriod.Daily, + new DateTime(2026, 3, 22, 0, 0, 0, DateTimeKind.Utc), + ex + ); + + // Assert + Assert.Equal(ArchivePeriod.Daily, args.Period); + Assert.Same(ex, args.Exception); + } + + [Fact] + public void ArchiveJournalEntry_DefaultState() + { + // Arrange & Act + var entry = new ArchiveJournalEntry(); + + // Assert + Assert.NotNull(entry.SourceDirectories); + Assert.Empty(entry.SourceDirectories); + Assert.NotNull(entry.DestinationResults); + Assert.Empty(entry.DestinationResults); + Assert.Null(entry.FailureReason); + Assert.Equal(DateTime.MaxValue, entry.CompletedAt); + } +} diff --git a/Projects/UOContent.Tests/Tests/WorldSaves/ManagedArchiveTests.cs b/Projects/UOContent.Tests/Tests/WorldSaves/ManagedArchiveTests.cs new file mode 100644 index 000000000..b5a2c0dc9 --- /dev/null +++ b/Projects/UOContent.Tests/Tests/WorldSaves/ManagedArchiveTests.cs @@ -0,0 +1,325 @@ +using System; +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; +using Server.Compression; +using Xunit; + +namespace Server.Tests; + +[Collection("Sequential UOContent Tests")] +public class ManagedArchiveTests : IDisposable +{ + private readonly string _testDir; + + public ManagedArchiveTests() + { + _testDir = Path.Combine(Path.GetTempPath(), $"modernuo-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_testDir); + } + + public void Dispose() + { + if (Directory.Exists(_testDir)) + { + Directory.Delete(_testDir, true); + } + } + + private string CreateTestDirectory(string name, params (string fileName, string content)[] files) + { + var dir = Path.Combine(_testDir, name); + Directory.CreateDirectory(dir); + + foreach (var (fileName, content) in files) + { + var filePath = Path.Combine(dir, fileName); + var fileDir = Path.GetDirectoryName(filePath); + if (fileDir != null) + { + Directory.CreateDirectory(fileDir); + } + + File.WriteAllText(filePath, content); + } + + return dir; + } + + [Fact] + public void CreateTarZstd_SingleDirectory_RoundTrips() + { + // Arrange + var sourceDir = CreateTestDirectory("source", + ("file1.txt", "Hello World"), + ("file2.bin", "Binary content here"), + ("subdir/nested.txt", "Nested file") + ); + + var archivePath = Path.Combine(_testDir, "test.tar.zst"); + + // Act + var entryCount = ManagedArchive.CreateTarZstd( + [sourceDir], archivePath, _testDir, compressionLevel: 1 + ); + + // Assert + Assert.True(entryCount > 0); + Assert.True(File.Exists(archivePath)); + Assert.True(new FileInfo(archivePath).Length > 0); + } + + [Fact] + public void CreateTarZstd_MultipleDirectories_IncludesAll() + { + // Arrange + var dir1 = CreateTestDirectory("backup1", + ("items.bin", "items data"), + ("items.idx", "items index") + ); + var dir2 = CreateTestDirectory("backup2", + ("mobiles.bin", "mobiles data"), + ("mobiles.idx", "mobiles index") + ); + + var archivePath = Path.Combine(_testDir, "multi.tar.zst"); + + // Act + var entryCount = ManagedArchive.CreateTarZstd( + [dir1, dir2], archivePath, _testDir, compressionLevel: 1 + ); + + // Assert + Assert.Equal(4, entryCount); + } + + [Fact] + public void ExtractTarZstd_RecreatesFiles() + { + // Arrange + var sourceDir = CreateTestDirectory("source", + ("data.txt", "Test data content"), + ("config/settings.json", "{\"key\":\"value\"}") + ); + + var archivePath = Path.Combine(_testDir, "extract-test.tar.zst"); + ManagedArchive.CreateTarZstd([sourceDir], archivePath, _testDir, compressionLevel: 1); + + var extractDir = Path.Combine(_testDir, "extracted"); + + // Act + var success = ManagedArchive.ExtractTarZstd(archivePath, extractDir); + + // Assert + Assert.True(success); + Assert.True(Directory.Exists(extractDir)); + + // Verify extracted content matches + var extractedDataPath = Path.Combine(extractDir, "source", "data.txt"); + Assert.True(File.Exists(extractedDataPath)); + Assert.Equal("Test data content", File.ReadAllText(extractedDataPath)); + + var extractedConfigPath = Path.Combine(extractDir, "source", "config", "settings.json"); + Assert.True(File.Exists(extractedConfigPath)); + Assert.Equal("{\"key\":\"value\"}", File.ReadAllText(extractedConfigPath)); + } + + [Fact] + public void CountEntries_MatchesCreateCount() + { + // Arrange + var sourceDir = CreateTestDirectory("source", + ("a.txt", "a"), + ("b.txt", "b"), + ("c/d.txt", "d"), + ("c/e.txt", "e") + ); + + var archivePath = Path.Combine(_testDir, "count-test.tar.zst"); + var createCount = ManagedArchive.CreateTarZstd( + [sourceDir], archivePath, _testDir, compressionLevel: 1 + ); + + // Act + var verifyCount = ManagedArchive.CountEntries(archivePath); + + // Assert + Assert.Equal(createCount, verifyCount); + } + + [Fact] + public void CreateTarZstd_EmptyDirectory_ReturnsZero() + { + // Arrange + var sourceDir = Path.Combine(_testDir, "empty"); + Directory.CreateDirectory(sourceDir); + + var archivePath = Path.Combine(_testDir, "empty.tar.zst"); + + // Act + var entryCount = ManagedArchive.CreateTarZstd( + [sourceDir], archivePath, _testDir, compressionLevel: 1 + ); + + // Assert + Assert.Equal(0, entryCount); + } + + [Fact] + public void CreateTarZstd_NonexistentSource_SkipsGracefully() + { + // Arrange + var archivePath = Path.Combine(_testDir, "nonexistent.tar.zst"); + + // Act + var entryCount = ManagedArchive.CreateTarZstd( + [Path.Combine(_testDir, "does-not-exist")], archivePath, _testDir, compressionLevel: 1 + ); + + // Assert + Assert.Equal(0, entryCount); + } + + [Fact] + public void CreateTarZstd_TempFileCleanedUp() + { + // Arrange + var sourceDir = CreateTestDirectory("source", ("file.txt", "content")); + var archivePath = Path.Combine(_testDir, "cleanup-test.tar.zst"); + + // Act + ManagedArchive.CreateTarZstd([sourceDir], archivePath, _testDir, compressionLevel: 1); + + // Assert - no .tmp file should remain + Assert.False(File.Exists(archivePath + ".tmp")); + Assert.True(File.Exists(archivePath)); + } + + [Fact] + public void ExtractTarZstd_InvalidFile_ReturnsFalse() + { + // Arrange + var badFile = Path.Combine(_testDir, "bad.tar.zst"); + File.WriteAllText(badFile, "this is not a valid archive"); + var extractDir = Path.Combine(_testDir, "extract-bad"); + + // Act + var result = ManagedArchive.ExtractTarZstd(badFile, extractDir); + + // Assert + Assert.False(result); + } + + [Fact] + public void CountEntries_InvalidFile_ReturnsNegative() + { + // Arrange + var badFile = Path.Combine(_testDir, "bad.tar.zst"); + File.WriteAllText(badFile, "not a valid archive"); + + // Act + var count = ManagedArchive.CountEntries(badFile); + + // Assert + Assert.Equal(-1, count); + } + + [Theory] + [InlineData(1)] + [InlineData(3)] + [InlineData(10)] + public void CreateTarZstd_DifferentCompressionLevels_AllWork(int level) + { + // Arrange + var sourceDir = CreateTestDirectory($"level-{level}", ("data.txt", new string('X', 10000))); + var archivePath = Path.Combine(_testDir, $"level-{level}.tar.zst"); + + // Act + var entryCount = ManagedArchive.CreateTarZstd( + [sourceDir], archivePath, _testDir, compressionLevel: level + ); + + // Assert + Assert.Equal(1, entryCount); + Assert.True(File.Exists(archivePath)); + + // Verify we can extract it back + var extractDir = Path.Combine(_testDir, $"extract-level-{level}"); + Assert.True(ManagedArchive.ExtractTarZstd(archivePath, extractDir)); + } + + [Fact] + public void ExtractTarGz_RoundTrips() + { + // Arrange — create a .tar.gz using built-in .NET APIs + var sourceDir = CreateTestDirectory("gz-source", + ("data.txt", "GZip test data"), + ("sub/nested.txt", "Nested content") + ); + + var archivePath = Path.Combine(_testDir, "test.tar.gz"); + + using (var fileStream = new FileStream(archivePath, FileMode.Create)) + using (var gzStream = new GZipStream(fileStream, System.IO.Compression.CompressionLevel.Fastest)) + using (var tarWriter = new TarWriter(gzStream, TarEntryFormat.Pax, leaveOpen: true)) + { + foreach (var file in new DirectoryInfo(sourceDir).EnumerateFiles("*", SearchOption.AllDirectories)) + { + var entryName = Path.GetRelativePath(sourceDir, file.FullName).Replace('\\', '/'); + tarWriter.WriteEntry(file.FullName, entryName); + } + } + + var extractDir = Path.Combine(_testDir, "gz-extracted"); + + // Act + var success = ManagedArchive.ExtractTarGz(archivePath, extractDir); + + // Assert + Assert.True(success); + Assert.Equal("GZip test data", File.ReadAllText(Path.Combine(extractDir, "data.txt"))); + Assert.Equal("Nested content", File.ReadAllText(Path.Combine(extractDir, "sub", "nested.txt"))); + } + + [Fact] + public void ExtractTar_PlainTarRoundTrips() + { + // Arrange — create a plain .tar using built-in .NET APIs + var sourceDir = CreateTestDirectory("tar-source", + ("plain.txt", "Plain tar test data") + ); + + var archivePath = Path.Combine(_testDir, "test.tar"); + + using (var fileStream = new FileStream(archivePath, FileMode.Create)) + using (var tarWriter = new TarWriter(fileStream, TarEntryFormat.Pax, leaveOpen: true)) + { + foreach (var file in new DirectoryInfo(sourceDir).EnumerateFiles("*", SearchOption.AllDirectories)) + { + var entryName = Path.GetRelativePath(sourceDir, file.FullName).Replace('\\', '/'); + tarWriter.WriteEntry(file.FullName, entryName); + } + } + + var extractDir = Path.Combine(_testDir, "tar-extracted"); + + // Act + var success = ManagedArchive.ExtractTar(archivePath, extractDir); + + // Assert + Assert.True(success); + Assert.Equal("Plain tar test data", File.ReadAllText(Path.Combine(extractDir, "plain.txt"))); + } + + [Fact] + public void ExtractTarGz_InvalidFile_ReturnsFalse() + { + // Arrange + var badFile = Path.Combine(_testDir, "bad.tar.gz"); + File.WriteAllText(badFile, "not a valid gzip archive"); + var extractDir = Path.Combine(_testDir, "extract-bad-gz"); + + // Act & Assert + Assert.False(ManagedArchive.ExtractTarGz(badFile, extractDir)); + } +} diff --git a/Projects/UOContent/Compression/ManagedArchive.cs b/Projects/UOContent/Compression/ManagedArchive.cs new file mode 100644 index 000000000..32142d9e4 --- /dev/null +++ b/Projects/UOContent/Compression/ManagedArchive.cs @@ -0,0 +1,279 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: ManagedArchive.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.Collections.Generic; +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; +using Server.Logging; +using ZstdNet; + +namespace Server.Compression; + +/// +/// Cross-platform archive creation and extraction using System.Formats.Tar + ZstdNet (native libzstd). +/// Single-pass streaming: TarWriter → CompressionStream → FileStream. +/// +public static class ManagedArchive +{ + private static readonly ILogger logger = LogFactory.GetLogger(typeof(ManagedArchive)); + + /// + /// Creates a .tar.zst archive from one or more source directories. + /// Writes to a temp file first, then atomically renames to the destination. + /// + /// Directories to include in the archive. + /// Final .tar.zst file path. + /// Base path for computing relative entry names. + /// ZSTD compression level (1-22, default 3). + /// The number of entries written, or -1 on failure. + public static int CreateTarZstd( + IEnumerable sourcePaths, + string destinationFile, + string relativeTo, + int compressionLevel = 3) + { + var tempFile = destinationFile + ".tmp"; + + try + { + new FileInfo(destinationFile).EnsureDirectory(); + + using var fileStream = new FileStream( + tempFile, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920 + ); + + using var compressionOptions = new CompressionOptions(compressionLevel); + using var zstdStream = new CompressionStream(fileStream, compressionOptions); + using var tarWriter = new TarWriter(zstdStream, TarEntryFormat.Pax, leaveOpen: true); + + var entryCount = 0; + long totalBytes = 0; + + foreach (var sourcePath in sourcePaths) + { + if (!Directory.Exists(sourcePath)) + { + logger.Warning("Source path does not exist, skipping: {Path}", sourcePath); + continue; + } + + var sourceDir = new DirectoryInfo(sourcePath); + var relativeName = Path.GetRelativePath(relativeTo, sourcePath); + + foreach (var file in sourceDir.EnumerateFiles("*", SearchOption.AllDirectories)) + { + var entryName = Path.Combine(relativeName, Path.GetRelativePath(sourcePath, file.FullName)) + .Replace('\\', '/'); + + tarWriter.WriteEntry(file.FullName, entryName); + entryCount++; + totalBytes += file.Length; + + if (entryCount % 100 == 0) + { + logger.Debug( + "Archive progress: {Count} entries, {Size:F1} MB", + entryCount, + totalBytes / 1048576.0 + ); + } + } + } + + // Flush tar → zstd → file + tarWriter.Dispose(); + zstdStream.Dispose(); + fileStream.Dispose(); + + // Atomic rename on same filesystem + if (File.Exists(destinationFile)) + { + File.Delete(destinationFile); + } + + File.Move(tempFile, destinationFile); + + logger.Information( + "Created archive at {Path} ({Count} entries, {Size:F1} MB)", + destinationFile, + entryCount, + totalBytes / 1048576.0 + ); + + return entryCount; + } + catch (Exception ex) + { + logger.Error(ex, "Failed to create archive at {Path}", destinationFile); + + try + { + if (File.Exists(tempFile)) + { + File.Delete(tempFile); + } + } + catch + { + // Best-effort cleanup + } + + return -1; + } + } + + /// + /// Extracts a .tar.zst archive to a directory. + /// + public static bool ExtractTarZstd(string archiveFile, string outputDirectory) + { + try + { + Directory.CreateDirectory(outputDirectory); + + using var fileStream = new FileStream( + archiveFile, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920 + ); + + using var zstdStream = new DecompressionStream(fileStream); + return ExtractTarFromStream(zstdStream, outputDirectory); + } + catch (Exception ex) + { + logger.Error(ex, "Failed to extract archive {File}", archiveFile); + return false; + } + } + + /// + /// Extracts a .tar.gz archive to a directory. + /// + public static bool ExtractTarGz(string archiveFile, string outputDirectory) + { + try + { + Directory.CreateDirectory(outputDirectory); + + using var fileStream = new FileStream( + archiveFile, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920 + ); + + using var gzStream = new GZipStream(fileStream, CompressionMode.Decompress); + return ExtractTarFromStream(gzStream, outputDirectory); + } + catch (Exception ex) + { + logger.Error(ex, "Failed to extract archive {File}", archiveFile); + return false; + } + } + + /// + /// Extracts a plain .tar archive to a directory. + /// + public static bool ExtractTar(string archiveFile, string outputDirectory) + { + try + { + Directory.CreateDirectory(outputDirectory); + + using var fileStream = new FileStream( + archiveFile, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920 + ); + + return ExtractTarFromStream(fileStream, outputDirectory); + } + catch (Exception ex) + { + logger.Error(ex, "Failed to extract archive {File}", archiveFile); + return false; + } + } + + private static bool ExtractTarFromStream(Stream stream, string outputDirectory) + { + using var tarReader = new TarReader(stream); + + while (tarReader.GetNextEntry() is { } entry) + { + var destPath = Path.Combine(outputDirectory, entry.Name); + + var destDir = Path.GetDirectoryName(destPath); + if (destDir != null) + { + Directory.CreateDirectory(destDir); + } + + if (entry.EntryType is TarEntryType.RegularFile or TarEntryType.V7RegularFile) + { + entry.ExtractToFile(destPath, overwrite: true); + } + } + + return true; + } + + /// + /// Counts entries in a .tar.zst archive without extracting content. + /// Used for verification after archive creation. + /// + public static int CountEntries(string archiveFile) + { + try + { + using var fileStream = new FileStream( + archiveFile, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920 + ); + + using var zstdStream = new DecompressionStream(fileStream); + using var tarReader = new TarReader(zstdStream); + + var count = 0; + while (tarReader.GetNextEntry() is not null) + { + count++; + } + + return count; + } + catch (Exception ex) + { + logger.Error(ex, "Failed to count entries in archive {File}", archiveFile); + return -1; + } + } +} diff --git a/Projects/UOContent/Compression/TarArchive.cs b/Projects/UOContent/Compression/TarArchive.cs deleted file mode 100755 index 206c7eaa2..000000000 --- a/Projects/UOContent/Compression/TarArchive.cs +++ /dev/null @@ -1,122 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.IO.Compression; -using System.Net.Http; -using Server.Text; - -namespace Server.Compression -{ - public static class TarArchive - { - private const string _libArchiveWindowsUrl = @"https://www.libarchive.org/downloads/libarchive-v3.5.2-win64.zip"; - private static string _pathToTar; - - private static string GetPathToTar() - { - if (!Core.IsWindows || File.Exists(@"C:\Windows\system32\tar.exe")) - { - return "tar"; - } - - var tarPath = Path.Combine(Core.BaseDirectory, "bsdtar/bsdtar.exe"); - return File.Exists(tarPath) ? tarPath : DownloadTarForWindows(); - } - - private static int RunTar(string arguments, string compressionProgramPath) - { - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = _pathToTar, - Arguments = arguments - } - }; - - if (compressionProgramPath != null) - { - var envPaths = $"{process.StartInfo.EnvironmentVariables["PATH"]}:{compressionProgramPath}"; - process.StartInfo.EnvironmentVariables["PATH"] = envPaths; - } - - process.Start(); - process.WaitForExit(); - - return process.ExitCode; - } - - private static string DownloadTarForWindows() - { - var tempDir = PathUtility.EnsureRandomPath(Path.GetTempPath()); - - var libarchiveFile = Path.Combine(tempDir, "libarchive.zip"); - // This isn't called often so we don't need to optimize - using (var hc = new HttpClient()) - { - var result = hc.Send(new HttpRequestMessage(HttpMethod.Get, new Uri(_libArchiveWindowsUrl))); - using var stream = result.Content.ReadAsStream(); - using var fs = new FileStream(libarchiveFile, FileMode.Create, FileAccess.Write, FileShare.None); - stream.CopyTo(fs); - } - - ZipFile.ExtractToDirectory(libarchiveFile, tempDir); - var libArchivePath = Path.Combine(tempDir, "libarchive"); - PathUtility.MoveDirectoryContents(Path.Combine(libArchivePath, "bin"), Path.Combine(Core.BaseDirectory, "bsdtar")); - File.Delete(libarchiveFile); - - return Path.Combine(Core.BaseDirectory, "bsdtar/bsdtar.exe"); - } - - public static bool ExtractToDirectory( - string fileNamePath, - string outputDirectory, - string compressCommand = null, - string compressionProgramPath = null - ) - { - _pathToTar ??= GetPathToTar(); - - var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : ""; - var arguments = $"{useExternalCompression} -xf \"{fileNamePath}\" -C \"{outputDirectory}\""; - - return RunTar(arguments, compressionProgramPath) == 0; - } - - public static bool CreateFromPaths( - IEnumerable paths, - string destinationArchiveFileName, - string relativeTo, - string compressCommand = null, - string compressionProgramPath = null - ) - { - _pathToTar ??= GetPathToTar(); - - new FileInfo(destinationArchiveFileName).EnsureDirectory(); - - using var sb = ValueStringBuilder.Create(); - var i = 0; - foreach (var path in paths) - { - var relativePath = Path.GetRelativePath(relativeTo, path); - if (i++ > 0) - { - sb.Append($" \"{relativePath}\""); - } - else - { - sb.Append($"\"{relativePath}\""); - } - } - var pathsToCompress = sb.ToString(); - - var tarFlags = compressCommand == null ? "-acf" : "-cf"; - var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : ""; - var arguments = $"{useExternalCompression}{tarFlags} \"{destinationArchiveFileName}\" -C \"{relativeTo}\" {pathsToCompress}"; - - return RunTar(arguments, compressionProgramPath) == 0; - } - } -} diff --git a/Projects/UOContent/Compression/ZstdArchive.cs b/Projects/UOContent/Compression/ZstdArchive.cs deleted file mode 100755 index 35fd533d5..000000000 --- a/Projects/UOContent/Compression/ZstdArchive.cs +++ /dev/null @@ -1,95 +0,0 @@ -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Reflection; - -namespace Server.Compression -{ - public static class ZstdArchive - { - private static readonly string _pathToZstd = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory?.FullName; - - public static bool ExtractToDirectory(string fileNamePath, string outputDirectory) - { - // bsdtar has a bug and hangs, so we are doing it in two steps. - if (Core.IsWindows) - { - var tempDir = PathUtility.EnsureRandomPath(Path.GetTempPath()); - var tempTarArchive = Path.Combine(tempDir, "temp-file.tar"); - - try - { - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = Path.Combine(_pathToZstd, "zstd.exe"), - Arguments = $"-q -d \"{fileNamePath}\" -o \"{tempTarArchive}\"" - } - }; - - process.Start(); - process.WaitForExit(); - - return process.ExitCode == 0 && TarArchive.ExtractToDirectory(tempTarArchive, outputDirectory); - } - catch - { - return false; - } - finally - { - File.Delete(tempTarArchive); - } - } - - return TarArchive.ExtractToDirectory(fileNamePath, outputDirectory, "zstd -q -d", _pathToZstd); - } - - public static bool CreateFromPaths( - IEnumerable paths, - string destinationArchiveFileName, - string relativeTo - ) - { - // bsdtar has a bug and hangs, so we are doing it in two steps. - if (Core.IsWindows) - { - var tempDir = PathUtility.EnsureRandomPath(Path.GetTempPath()); - var tempTarArchive = Path.Combine(tempDir, "temp-file.tar"); - - try - { - if (!TarArchive.CreateFromPaths(paths, tempTarArchive, relativeTo)) - { - return false; - } - - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = Path.Combine(_pathToZstd, "zstd.exe"), - Arguments = $"-q \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\"" - } - }; - - process.Start(); - process.WaitForExit(); - - return process.ExitCode == 0; - } - catch - { - return false; - } - finally - { - Directory.Delete(tempDir, true); - } - } - - return TarArchive.CreateFromPaths(paths, destinationArchiveFileName, relativeTo, "zstd -q", _pathToZstd); - } - } -} diff --git a/Projects/UOContent/UOContent.csproj b/Projects/UOContent/UOContent.csproj index 54fbf6cff..22b81d23e 100644 --- a/Projects/UOContent/UOContent.csproj +++ b/Projects/UOContent/UOContent.csproj @@ -29,8 +29,7 @@ - - + @@ -45,7 +44,7 @@ - + diff --git a/Projects/UOContent/World Saves/ArchiveDestinationRegistry.cs b/Projects/UOContent/World Saves/ArchiveDestinationRegistry.cs new file mode 100644 index 000000000..74219ffea --- /dev/null +++ b/Projects/UOContent/World Saves/ArchiveDestinationRegistry.cs @@ -0,0 +1,34 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: ArchiveDestinationRegistry.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System.Collections.Generic; + +namespace Server.Saves; + +/// +/// Registration point for archive destinations. Register during Configure() phase. +/// No locks needed — registration is single-threaded, reading happens on one +/// ThreadPool thread at a time (guarded by the archive concurrency flag). +/// +public static class ArchiveDestinationRegistry +{ + private static readonly List _destinations = []; + + public static IReadOnlyList Destinations => _destinations; + + public static void Register(IArchiveDestination destination) => _destinations.Add(destination); + + public static void Unregister(IArchiveDestination destination) => _destinations.Remove(destination); +} diff --git a/Projects/UOContent/World Saves/ArchiveEnums.cs b/Projects/UOContent/World Saves/ArchiveEnums.cs new file mode 100644 index 000000000..fe412d4b9 --- /dev/null +++ b/Projects/UOContent/World Saves/ArchiveEnums.cs @@ -0,0 +1,23 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: ArchiveEnums.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +namespace Server.Saves; + +public enum ArchivePeriod +{ + Hourly, + Daily, + Monthly +} diff --git a/Projects/UOContent/World Saves/ArchiveEventArgs.cs b/Projects/UOContent/World Saves/ArchiveEventArgs.cs new file mode 100644 index 000000000..b1f8db63b --- /dev/null +++ b/Projects/UOContent/World Saves/ArchiveEventArgs.cs @@ -0,0 +1,40 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: ArchiveEventArgs.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; + +namespace Server.Saves; + +public class ArchiveCompletedEventArgs( + string archiveFilePath, + ArchivePeriod period, + DateTime rangeStart, + long fileSizeBytes, + double elapsedSeconds +) +{ + public string ArchiveFilePath { get; } = archiveFilePath; + public ArchivePeriod Period { get; } = period; + public DateTime RangeStart { get; } = rangeStart; + public long FileSizeBytes { get; } = fileSizeBytes; + public double ElapsedSeconds { get; } = elapsedSeconds; +} + +public class ArchiveFailedEventArgs(ArchivePeriod period, DateTime rangeStart, Exception exception) +{ + public ArchivePeriod Period { get; } = period; + public DateTime RangeStart { get; } = rangeStart; + public Exception Exception { get; } = exception; +} diff --git a/Projects/UOContent/World Saves/ArchiveJournal.cs b/Projects/UOContent/World Saves/ArchiveJournal.cs new file mode 100644 index 000000000..0ec7f0e28 --- /dev/null +++ b/Projects/UOContent/World Saves/ArchiveJournal.cs @@ -0,0 +1,269 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: ArchiveJournal.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.Collections.Generic; +using System.IO; +using Server.Json; +using Server.Logging; + +namespace Server.Saves; + +public enum ArchiveOperationState +{ + Started, + Archived, + Distributed, + Completed, + Failed +} + +public class ArchiveJournalEntry +{ + public string Id { get; set; } + public ArchivePeriod Period { get; set; } + public DateTime RangeStart { get; set; } + public ArchiveOperationState State { get; set; } + public string ArchiveFile { get; set; } + public string TempFile { get; set; } + public List SourceDirectories { get; set; } = []; + public int EntryCount { get; set; } + public long TotalSourceBytes { get; set; } + public long ArchiveBytes { get; set; } + public DateTime StartedAt { get; set; } + public DateTime CompletedAt { get; set; } = DateTime.MaxValue; + public string FailureReason { get; set; } + public Dictionary DestinationResults { get; set; } = new(); +} + +public class ArchiveJournalData +{ + public List Operations { get; set; } = []; +} + +public static class ArchiveJournal +{ + private const int MaxJournalEntries = 100; + + private static readonly ILogger logger = LogFactory.GetLogger(typeof(ArchiveJournal)); + + private static string _journalPath; + private static ArchiveJournalData _data = new(); + + public static IReadOnlyList Operations => _data.Operations; + + public static void Configure(string archivePath) + { + _journalPath = Path.Combine(archivePath, ".archive-journal.json"); + _data = new ArchiveJournalData(); + Load(); + } + + private static void Load() + { + try + { + if (File.Exists(_journalPath)) + { + _data = JsonConfig.Deserialize(_journalPath) ?? new ArchiveJournalData(); + } + } + catch (Exception ex) + { + logger.Error(ex, "Failed to load archive journal, starting fresh"); + _data = new ArchiveJournalData(); + } + } + + private static void Save() + { + try + { + var tempPath = _journalPath + ".tmp"; + JsonConfig.Serialize(tempPath, _data); + + if (File.Exists(_journalPath)) + { + File.Delete(_journalPath); + } + + File.Move(tempPath, _journalPath); + } + catch (Exception ex) + { + logger.Error(ex, "Failed to save archive journal"); + } + } + + private static void Prune() + { + if (_data.Operations.Count > MaxJournalEntries) + { + // Remove oldest completed/failed entries beyond the limit + var toRemove = _data.Operations.Count - MaxJournalEntries; + for (var i = _data.Operations.Count - 1; i >= 0 && toRemove > 0; i--) + { + if (_data.Operations[i].State is ArchiveOperationState.Completed or ArchiveOperationState.Failed) + { + _data.Operations.RemoveAt(i); + toRemove--; + } + } + } + } + + public static ArchiveJournalEntry BeginOperation( + ArchivePeriod period, + DateTime rangeStart, + string tempFile, + string archiveFile, + List sourceDirectories) + { + var entry = new ArchiveJournalEntry + { + Id = $"{rangeStart:O}-{period.ToString().ToLowerInvariant()}", + Period = period, + RangeStart = rangeStart, + State = ArchiveOperationState.Started, + TempFile = tempFile, + ArchiveFile = archiveFile, + SourceDirectories = sourceDirectories, + StartedAt = Core.Now + }; + + _data.Operations.Add(entry); + Prune(); + Save(); + return entry; + } + + public static void RecordArchived(ArchiveJournalEntry entry, int entryCount, long archiveBytes) + { + entry.State = ArchiveOperationState.Archived; + entry.EntryCount = entryCount; + entry.ArchiveBytes = archiveBytes; + entry.TempFile = null; // Temp file has been renamed to final + Save(); + } + + public static void RecordDistributed(ArchiveJournalEntry entry, Dictionary results) + { + entry.State = ArchiveOperationState.Distributed; + entry.DestinationResults = results; + Save(); + } + + public static void RecordCompleted(ArchiveJournalEntry entry) + { + entry.State = ArchiveOperationState.Completed; + entry.CompletedAt = Core.Now; + Save(); + } + + public static void RecordFailure(ArchiveJournalEntry entry, string reason) + { + entry.State = ArchiveOperationState.Failed; + entry.FailureReason = reason; + entry.CompletedAt = Core.Now; + Save(); + } + + /// + /// Recovers from interrupted operations on startup. + /// Handles missing files/directories gracefully (e.g. admin cleaned up archives folder). + /// + public static void RecoverInterrupted() + { + var recovered = false; + + foreach (var entry in _data.Operations) + { + switch (entry.State) + { + case ArchiveOperationState.Started: + { + // Archive creation was interrupted. Clean up temp file if it exists. + if (entry.TempFile != null) + { + try + { + if (File.Exists(entry.TempFile)) + { + File.Delete(entry.TempFile); + } + } + catch (Exception ex) + { + logger.Warning(ex, "Failed to clean up temp file {File}", entry.TempFile); + } + } + + logger.Warning( + "Recovered interrupted archive operation {Id} (was in Started state)", + entry.Id + ); + entry.State = ArchiveOperationState.Failed; + entry.FailureReason = "Interrupted during archive creation"; + entry.CompletedAt = Core.Now; + recovered = true; + break; + } + case ArchiveOperationState.Archived: + { + // Archive was created but not distributed. + // If the archive file was deleted (admin cleanup), mark as failed. + if (entry.ArchiveFile != null && !File.Exists(entry.ArchiveFile)) + { + logger.Warning( + "Archive file for operation {Id} no longer exists (manually deleted?). Marking failed.", + entry.Id + ); + entry.State = ArchiveOperationState.Failed; + entry.FailureReason = "Archive file missing on recovery"; + entry.CompletedAt = Core.Now; + recovered = true; + } + else + { + logger.Warning( + "Found unfinished archive operation {Id} (Archived but not distributed). " + + "Will re-attempt distribution on next cycle.", + entry.Id + ); + } + + break; + } + case ArchiveOperationState.Distributed: + { + // Sources should have been pruned but weren't. + logger.Information( + "Completing interrupted operation {Id} (was Distributed, pruning sources)", + entry.Id + ); + entry.State = ArchiveOperationState.Completed; + entry.CompletedAt = Core.Now; + recovered = true; + break; + } + } + } + + if (recovered) + { + Save(); + } + } +} diff --git a/Projects/UOContent/World Saves/AutoArchive.cs b/Projects/UOContent/World Saves/AutoArchive.cs index 1c25d34cf..aca0f75a9 100755 --- a/Projects/UOContent/World Saves/AutoArchive.cs +++ b/Projects/UOContent/World Saves/AutoArchive.cs @@ -1,3 +1,18 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: AutoArchive.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + using System; using System.Collections.Generic; using System.Diagnostics; @@ -7,336 +22,833 @@ using System.Threading; using CommunityToolkit.HighPerformance; using Server.Compression; using Server.Logging; +using Server.Text; -namespace Server.Saves +namespace Server.Saves; + +public static class AutoArchive { - public enum ArchivePeriod + private static readonly ILogger logger = LogFactory.GetLogger(typeof(AutoArchive)); + + private static string _tempArchivePath; + private static int _isArchiving; + private static DateTime _nextHourlyArchive; + private static DateTime _nextDailyArchive; + private static DateTime _nextMonthlyArchive; + private static bool _enablePruning; + private static int _compressionLevel; + private static bool _verifyArchives; + private static int _retryCount; + private static int _retryDelayMs; + private static int _backupMaxAgeDays; + + public static event Action ArchiveCompleted; + public static event Action ArchiveFailed; + + public static Action Archive { get; set; } + public static Action Prune { get; set; } + public static string ArchivePath { get; private set; } + public static string BackupPath { get; private set; } + public static string AutomaticBackupPath { get; private set; } + + public static DateTime NextHourlyArchive => _nextHourlyArchive; + public static DateTime NextDailyArchive => _nextDailyArchive; + public static DateTime NextMonthlyArchive => _nextMonthlyArchive; + + public static void Configure() { - Hourly, - Daily, - Monthly + var tempArchivePath = ServerConfiguration.GetSetting("autoArchive.tempArchivePath", "temp"); + _tempArchivePath = PathUtility.GetFullPath(tempArchivePath); + + var backupPath = ServerConfiguration.GetOrUpdateSetting("autoArchive.backupPath", "Backups"); + BackupPath = PathUtility.GetFullPath(backupPath); + AutomaticBackupPath = Path.Combine(BackupPath, "Automatic"); + + var archivePath = ServerConfiguration.GetOrUpdateSetting("autoArchive.archivePath", "Archives"); + ArchivePath = PathUtility.GetFullPath(archivePath); + + var useLocalArchives = ServerConfiguration.GetOrUpdateSetting("autoArchive.archiveLocally", true); + _enablePruning = ServerConfiguration.GetOrUpdateSetting("autoArchive.enableArchivePruning", true); + + _compressionLevel = ServerConfiguration.GetOrUpdateSetting("autoArchive.compressionLevel", 3); + _verifyArchives = ServerConfiguration.GetOrUpdateSetting("autoArchive.verifyArchives", true); + _retryCount = ServerConfiguration.GetOrUpdateSetting("autoArchive.retryCount", 3); + _retryDelayMs = ServerConfiguration.GetOrUpdateSetting("autoArchive.retryDelayMs", 500); + _backupMaxAgeDays = ServerConfiguration.GetOrUpdateSetting("autoArchive.backupMaxAge", 30); + + // Configurable retention + var hourlyRetention = ServerConfiguration.GetOrUpdateSetting("autoArchive.hourlyRetention", 24); + var dailyRetention = ServerConfiguration.GetOrUpdateSetting("autoArchive.dailyRetention", 30); + var monthlyRetention = ServerConfiguration.GetOrUpdateSetting("autoArchive.monthlyRetention", 12); + + // Register local filesystem destination + if (useLocalArchives) + { + ArchiveDestinationRegistry.Register( + new LocalArchiveDestination(hourlyRetention, dailyRetention, monthlyRetention) + ); + Archive = AutoArchiveLocally; + } + + if (_enablePruning) + { + Prune = PruneBackups; + } + + // Initialize journal and recover interrupted operations + ArchiveJournal.Configure(ArchivePath); + ArchiveJournal.RecoverInterrupted(); + + // Restores an archive file placed in the Saves folder + RestoreFromArchive(); + + // If no valid save data exists, offer to restore from backups/archives + if (!HasValidSaveData()) + { + PromptRestoreFromBackup(); + } + + DateTimeOffset now = Core.Now; + var date = now.Date; + _nextHourlyArchive = date.AddHours(now.Hour); + _nextDailyArchive = date; + _nextMonthlyArchive = date.AddDays(1 - now.Day); + + // Do a local archive rollup & prune + AutoArchiveLocally(); } - public enum CompressionFormat + public static void Initialize() { - None, - Zip, - GZip, - Zstd, + EventSink.WorldSavePostSnapshot += Backup; } - public static class AutoArchive + private const string BackupCompleteMarker = ".backup-complete"; + + private static void Backup(WorldSavePostSnapshotEventArgs args) { - private static readonly ILogger logger = LogFactory.GetLogger(typeof(AutoArchive)); - - private static string _tempArchivePath; - private static int _isArchiving; - private static DateTime _nextHourlyArchive; - private static DateTime _nextDailyArchive; - private static DateTime _nextMonthlyArchive; - private static CompressionFormat _compressionFormat; - private static bool _enablePruning; - - public static Action Archive { get; set; } - public static Action Prune { get; set; } - public static string ArchivePath { get; private set; } - public static string BackupPath { get; private set; } - public static string AutomaticBackupPath { get; private set; } - - public static void Configure() + if (!Directory.Exists(args.OldSavePath)) { - var tempArchivePath = ServerConfiguration.GetSetting("autoArchive.tempArchivePath", "temp"); - _tempArchivePath = PathUtility.GetFullPath(tempArchivePath); - - var backupPath = ServerConfiguration.GetOrUpdateSetting("autoArchive.backupPath", "Backups"); - BackupPath = PathUtility.GetFullPath(backupPath); - AutomaticBackupPath = Path.Combine(BackupPath, "Automatic"); - - var archivePath = ServerConfiguration.GetOrUpdateSetting("autoArchive.archivePath", "Archives"); - ArchivePath = PathUtility.GetFullPath(archivePath); - - var useLocalArchives = ServerConfiguration.GetOrUpdateSetting("autoArchive.archiveLocally", true); - _enablePruning = ServerConfiguration.GetOrUpdateSetting("autoArchive.enableArchivePruning", true); - - _compressionFormat = ServerConfiguration.GetOrUpdateSetting("autoArchive.compressionFormat", CompressionFormat.Zstd); - - if (useLocalArchives) - { - Archive = AutoArchiveLocally; - } - - if (_enablePruning) - { - Prune = PruneBackups; - } - - // Restores an archive file placed in the Saves folder. Supports all compression formats. - RestoreFromArchive(); - - DateTimeOffset now = Core.Now; - var date = now.Date; - _nextHourlyArchive = date.AddHours(now.Hour); - _nextDailyArchive = date; - _nextMonthlyArchive = date.AddDays(1 - now.Day); - - // Do a local archive rollup & prune - AutoArchiveLocally(); + return; } - public static void Initialize() + Directory.CreateDirectory(AutomaticBackupPath); + var backupPath = Path.Combine(AutomaticBackupPath, Utility.GetTimeStamp()); + PathUtility.MoveDirectoryContents(args.OldSavePath, backupPath); + + // Write a marker file so Rollup knows this backup is complete and not mid-write. + // This prevents archiving a partially-moved directory if Rollup runs concurrently. + File.WriteAllBytes(Path.Combine(backupPath, BackupCompleteMarker), []); + + logger.Information("Created backup at {Path}", backupPath); + + Archive?.Invoke(); + } + + private static void RestoreFromArchive() + { + var savePath = Path.Combine(Core.BaseDirectory, ServerConfiguration.GetSetting("world.savePath", "Saves")); + if (!Directory.Exists(savePath)) { - EventSink.WorldSavePostSnapshot += Backup; + return; } - private static void Backup(WorldSavePostSnapshotEventArgs args) + foreach (var file in PathsByTimestampName(savePath, true)) { - if (!Directory.Exists(args.OldSavePath)) + if (RestoreFromFile(file, savePath)) { - return; - } - - Directory.CreateDirectory(AutomaticBackupPath); - var backupPath = Path.Combine(AutomaticBackupPath, Utility.GetTimeStamp()); - PathUtility.MoveDirectoryContents(args.OldSavePath, backupPath); - - logger.Information("Created backup at {Path}", backupPath); - - Archive?.Invoke(); - } - - private static void RestoreFromArchive() - { - var savePath = Path.Combine(Core.BaseDirectory, ServerConfiguration.GetSetting("world.savePath", "Saves")); - if (!Directory.Exists(savePath)) - { - return; - } - - foreach (var file in PathsByTimestampName(savePath, true)) - { - if (RestoreFromFile(file, savePath)) - { - File.Delete(file); - break; - } - } - } - - private static bool RestoreFromFile(string file, string savePath) - { - var fi = new FileInfo(file); - var fileName = fi.Name; - - if (!TryGetDate(fileName[..fileName.IndexOfOrdinal(".")], out _)) - { - return false; - } - - logger.Information("Restoring latest world save from archive {File}", fileName); - - var tempPath = PathUtility.EnsureRandomPath(_tempArchivePath); - var successful = fileName.EndsWithOrdinal(".tar.zst") - ? ZstdArchive.ExtractToDirectory(fi.FullName, tempPath) - : TarArchive.ExtractToDirectory(fi.FullName, tempPath); - - if (!successful) - { - logger.Information("Failed to extract {File}", fi.Name); - return false; - } - - foreach (var folder in PathsByTimestampName(tempPath)) - { - Directory.Delete(savePath, true); - var dirInfo = new DirectoryInfo(folder); - logger.Information("Restoring backup {Directory}", dirInfo.Name); - PathUtility.MoveDirectoryContents(folder, savePath); + File.Delete(file); break; } + } + } - Directory.Delete(tempPath, true); + private static bool RestoreFromFile(string file, string savePath) + { + var fi = new FileInfo(file); + var fileName = fi.Name; + if (!TryGetDate(fileName[..fileName.IndexOfOrdinal(".")], out _)) + { + return false; + } + + logger.Information("Restoring world save from archive {File}", fileName); + + var tempPath = PathUtility.EnsureRandomPath(_tempArchivePath); + try + { + if (!ExtractArchive(fi.FullName, tempPath)) + { + return false; + } + + return PromptAndRestoreFromExtractedArchive(tempPath, savePath); + } + finally + { + CleanupTempDirectory(tempPath); + } + } + + /// + /// Checks if the Saves directory contains valid world save data. + /// + private static bool HasValidSaveData() + { + var savePath = Path.Combine(Core.BaseDirectory, ServerConfiguration.GetSetting("world.savePath", "Saves")); + if (!Directory.Exists(savePath)) + { + return false; + } + + // A valid save has SerializedTypes.db or at least one .bin/.idx file + if (File.Exists(Path.Combine(savePath, "SerializedTypes.db"))) + { return true; } - public static void PruneBackups() + foreach (var dir in Directory.EnumerateDirectories(savePath)) { - if (Directory.Exists(ArchivePath)) + foreach (var file in Directory.EnumerateFiles(dir)) { - // Maintain a total of 66 of the most recent archives - PruneLocalArchives(ArchivePeriod.Hourly, 24); - PruneLocalArchives(ArchivePeriod.Daily, 30); - PruneLocalArchives(ArchivePeriod.Monthly, 12); - } - - if (!Directory.Exists(AutomaticBackupPath)) - { - return; - } - - var allFolders = Directory.EnumerateDirectories(AutomaticBackupPath); - var threshold = Core.Now.AddMonths(-1); - - foreach (var folder in allFolders) - { - var dirName = new DirectoryInfo(folder).Name; - - if (!TryGetDate(dirName, out var date)) + if (file.EndsWithOrdinal(".bin") || file.EndsWithOrdinal(".idx")) { - continue; - } - - if (date < threshold) - { - logger.Information("Pruning old backup {Directory}", folder); - Directory.Delete(folder, true); + return true; } } } - private static void PruneLocalArchives(ArchivePeriod period, int minRetained) - { - var periodStr = period.ToString(); - var archivePath = Path.Combine(ArchivePath, periodStr); - if (!Directory.Exists(archivePath)) - { - return; - } + return false; + } - var periodLowerStr = periodStr.ToLowerInvariant(); - foreach (var archive in PathsByTimestampName(archivePath, true)) + /// + /// Presents an interactive console menu to restore from a backup directory or archive file. + /// Only called during Configure() when no valid save data exists. + /// At this point, Console.ReadLine() works directly (ConsoleInputHandler not yet initialized). + /// + private static void PromptRestoreFromBackup() + { + // Collect available restore points: backup directories and archive files + var restorePoints = new SortedDictionary( + new DescendingComparer() + ); + + // Scan backup directories + if (Directory.Exists(AutomaticBackupPath)) + { + foreach (var dir in Directory.EnumerateDirectories(AutomaticBackupPath)) { - if (minRetained > 0) + if (!File.Exists(Path.Combine(dir, BackupCompleteMarker))) { - minRetained--; continue; } - var fi = new FileInfo(archive); - logger.Information("Pruning {Period} archive {File}", periodLowerStr, fi.Name); - File.Delete(archive); + var dirName = new DirectoryInfo(dir).Name; + if (TryGetDate(dirName, out var date)) + { + restorePoints[date] = (dir, $"Backup: {dirName}", false); + } } } - public static void AutoArchiveLocally() + // Scan archive files (hourly, daily, monthly) + if (Directory.Exists(ArchivePath)) { - var date = Core.Now; - - if (date < _nextHourlyArchive && date < _nextDailyArchive && date < _nextMonthlyArchive) + foreach (var periodDir in Directory.EnumerateDirectories(ArchivePath)) { - return; - } - - if (Interlocked.CompareExchange(ref _isArchiving, 0, 1) == 1) - { - return; - } - - ThreadPool.QueueUserWorkItem( - now => + var periodName = new DirectoryInfo(periodDir).Name; + foreach (var file in Directory.EnumerateFiles(periodDir)) { + var fi = new FileInfo(file); + var nameWithoutExt = fi.Name; + var dotIndex = nameWithoutExt.IndexOfOrdinal('.'); + if (dotIndex > 0) + { + nameWithoutExt = nameWithoutExt[..dotIndex]; + } + + if (TryGetDate(nameWithoutExt, out var date)) + { + restorePoints[date] = (file, $"{periodName} archive: {fi.Name}", true); + } + } + } + } + + if (restorePoints.Count == 0) + { + return; + } + + // Build flat list from sorted dictionary + var entries = new List<(string path, string label, bool isArchive, DateTime date)>(); + foreach (var (date, (path, label, isArchive)) in restorePoints) + { + entries.Add((path, label, isArchive, date)); + } + + // Interactive paginated menu + const int pageSize = 20; + var page = 0; + var totalPages = (entries.Count + pageSize - 1) / pageSize; + using var sb = ValueStringBuilder.Create(); + + while (true) + { + Console.WriteLine(); + Utility.PushColor(ConsoleColor.Yellow); + Console.WriteLine("No world save data found."); + Utility.PopColor(); + Console.Write($"Available backups and archives ({entries.Count} total"); + if (totalPages > 1) + { + Console.Write($", page {page + 1}/{totalPages}"); + } + + Console.WriteLine("):"); + Console.WriteLine(); + + var start = page * pageSize; + var end = Math.Min(start + pageSize, entries.Count); + + for (var i = start; i < end; i++) + { + var (_, label, _, date) = entries[i]; + Console.Write($" {i + 1,3}) "); + Utility.PushColor(ConsoleColor.Cyan); + Console.Write($"{date:yyyy-MM-dd HH:mm:ss}"); + Utility.PopColor(); + Console.WriteLine($" {label}"); + } + + Console.WriteLine(); + sb.Reset(); + sb.Append("Enter number to restore"); + if (page < totalPages - 1) + { + sb.Append(", [n]ext page"); + } + + if (page > 0) + { + sb.Append(", [p]rev page"); + } + + sb.Append(", or Enter to start fresh: "); + Console.Write(sb.ToString()); + + var input = Console.ReadLine(); + + if (string.IsNullOrWhiteSpace(input)) + { + Console.WriteLine("Starting with no world save data."); + return; + } + + if (input.InsensitiveEquals("n") && page < totalPages - 1) + { + page++; + continue; + } + + if (input.InsensitiveEquals("p") && page > 0) + { + page--; + continue; + } + + if (int.TryParse(input, out var choice) && choice >= 1 && choice <= entries.Count) + { + var selected = entries[choice - 1]; + Console.Write("Restoring "); + Utility.PushColor(ConsoleColor.Green); + Console.Write(selected.label); + Utility.PopColor(); + Console.WriteLine("..."); + + var savePath = Path.Combine( + Core.BaseDirectory, + ServerConfiguration.GetSetting("world.savePath", "Saves") + ); + + if (selected.isArchive) + { + RestoreFromArchiveFile(selected.path, savePath); + } + else + { + RestoreFromBackupDirectory(selected.path, savePath); + } + + return; + } + + Console.WriteLine("Invalid selection. Try again."); + } + } + + private static void RestoreFromArchiveFile(string archiveFile, string savePath) + { + var tempPath = PathUtility.EnsureRandomPath(_tempArchivePath); + try + { + if (!ExtractArchive(archiveFile, tempPath)) + { + Utility.PushColor(ConsoleColor.Red); + Console.WriteLine($"Failed to extract {new FileInfo(archiveFile).Name}."); + Utility.PopColor(); + return; + } + + PromptAndRestoreFromExtractedArchive(tempPath, savePath); + } + finally + { + CleanupTempDirectory(tempPath); + } + } + + private static void RestoreFromBackupDirectory(string backupDir, string savePath) + { + if (Directory.Exists(savePath)) + { + Directory.Delete(savePath, true); + } + + Directory.CreateDirectory(savePath); + PathUtility.CopyDirectoryContents(backupDir, savePath); + logger.Information("Restored world save from backup {Directory}", new DirectoryInfo(backupDir).Name); + } + + /// + /// Extracts an archive file (.tar.zst, .tar.gz, .tar) to a directory. + /// + private static bool ExtractArchive(string archiveFile, string outputDirectory) + { + var fileName = new FileInfo(archiveFile).Name; + + if (fileName.EndsWithOrdinal(".tar.zst")) + { + return ManagedArchive.ExtractTarZstd(archiveFile, outputDirectory); + } + + if (fileName.EndsWithOrdinal(".tar.gz")) + { + return ManagedArchive.ExtractTarGz(archiveFile, outputDirectory); + } + + if (fileName.EndsWithOrdinal(".tar")) + { + return ManagedArchive.ExtractTar(archiveFile, outputDirectory); + } + + logger.Warning("Unsupported archive format: {File}", fileName); + return false; + } + + /// + /// Lists backup directories inside an extracted archive and lets the user choose which to restore. + /// If only one backup exists, restores it directly. + /// + private static bool PromptAndRestoreFromExtractedArchive(string extractedPath, string savePath) + { + var backups = new List<(DateTime date, string path, string name)>(); + foreach (var dir in Directory.EnumerateDirectories(extractedPath)) + { + var dirName = new DirectoryInfo(dir).Name; + if (TryGetDate(dirName, out var date)) + { + backups.Add((date, dir, dirName)); + } + } + + if (backups.Count == 0) + { + Utility.PushColor(ConsoleColor.Red); + Console.WriteLine("No valid backup directories found inside the archive."); + Utility.PopColor(); + return false; + } + + // Sort newest first + backups.Sort((a, b) => b.date.CompareTo(a.date)); + + string selectedPath; + + if (backups.Count == 1) + { + selectedPath = backups[0].path; + logger.Information("Restoring backup {Directory}", backups[0].name); + } + else + { + // Multiple backups — let the user choose + Console.WriteLine(); + Console.WriteLine($"Archive contains {backups.Count} backups:"); + Console.WriteLine(); + + for (var i = 0; i < backups.Count; i++) + { + Console.Write($" {i + 1,3}) "); + Utility.PushColor(ConsoleColor.Cyan); + Console.Write($"{backups[i].date:yyyy-MM-dd HH:mm:ss}"); + Utility.PopColor(); + Console.WriteLine($" {backups[i].name}"); + } + + Console.WriteLine(); + Console.Write("[1]> "); + var input = Console.ReadLine(); + + int choice; + if (string.IsNullOrWhiteSpace(input)) + { + choice = 1; // Default to most recent + } + else if (!int.TryParse(input, out choice) || choice < 1 || choice > backups.Count) + { + Console.WriteLine("Invalid selection, using most recent."); + choice = 1; + } + + selectedPath = backups[choice - 1].path; + logger.Information("Restoring backup {Directory}", backups[choice - 1].name); + } + + if (Directory.Exists(savePath)) + { + Directory.Delete(savePath, true); + } + + Directory.CreateDirectory(savePath); + PathUtility.MoveDirectoryContents(selectedPath, savePath); + return true; + } + + private static void CleanupTempDirectory(string tempPath) + { + try + { + if (Directory.Exists(tempPath)) + { + Directory.Delete(tempPath, true); + } + } + catch (Exception ex) + { + logger.Warning(ex, "Failed to clean up temp directory {Path}", tempPath); + } + } + + public static void PruneBackups() + { + if (Directory.Exists(ArchivePath)) + { + // Get retention counts from registered destinations + var hourlyRetention = GetMaxRetention(ArchivePeriod.Hourly); + var dailyRetention = GetMaxRetention(ArchivePeriod.Daily); + var monthlyRetention = GetMaxRetention(ArchivePeriod.Monthly); + + PruneLocalArchives(ArchivePeriod.Hourly, hourlyRetention); + PruneLocalArchives(ArchivePeriod.Daily, dailyRetention); + PruneLocalArchives(ArchivePeriod.Monthly, monthlyRetention); + } + + if (!Directory.Exists(AutomaticBackupPath)) + { + return; + } + + var allFolders = Directory.EnumerateDirectories(AutomaticBackupPath); + var threshold = Core.Now.AddDays(-_backupMaxAgeDays); + + foreach (var folder in allFolders) + { + // Skip backup directories that are still being written + if (!File.Exists(Path.Combine(folder, BackupCompleteMarker))) + { + continue; + } + + var dirName = new DirectoryInfo(folder).Name; + + if (!TryGetDate(dirName, out var date)) + { + continue; + } + + if (date < threshold) + { + logger.Information("Pruning old backup {Directory}", folder); + RetryFileOperation(() => Directory.Delete(folder, true)); + } + } + } + + private static int GetMaxRetention(ArchivePeriod period) + { + var max = 0; + for (var i = 0; i < ArchiveDestinationRegistry.Destinations.Count; i++) + { + var dest = ArchiveDestinationRegistry.Destinations[i]; + var count = dest.GetRetentionCount(period); + if (count > max) + { + max = count; + } + } + + return max; + } + + private static void PruneLocalArchives(ArchivePeriod period, int minRetained) + { + var periodStr = period.ToString(); + var archivePath = Path.Combine(ArchivePath, periodStr); + if (!Directory.Exists(archivePath)) + { + return; + } + + var periodLowerStr = periodStr.ToLowerInvariant(); + foreach (var archive in PathsByTimestampName(archivePath, true)) + { + if (minRetained > 0) + { + minRetained--; + continue; + } + + var fi = new FileInfo(archive); + logger.Information("Pruning {Period} archive {File}", periodLowerStr, fi.Name); + RetryFileOperation(() => File.Delete(archive)); + } + } + + public static void AutoArchiveLocally() + { + var date = Core.Now; + + if (date < _nextHourlyArchive && date < _nextDailyArchive && date < _nextMonthlyArchive) + { + return; + } + + // Fixed: was CompareExchange(ref _isArchiving, 0, 1) which never guarded + if (Interlocked.CompareExchange(ref _isArchiving, 1, 0) == 1) + { + logger.Debug("Archive operation already in progress, skipping this cycle"); + return; + } + + ThreadPool.QueueUserWorkItem( + _ => + { + try + { + // Use current time for schedule checks (not the captured time from the caller) + // to avoid re-processing periods that elapsed during a long archive operation. + var now = Core.Now; + if (now >= _nextHourlyArchive) { Rollup(ArchivePeriod.Hourly); - _nextHourlyArchive = _nextHourlyArchive.AddHours(1); + AdvanceSchedule(ref _nextHourlyArchive, now, static dt => dt.AddHours(1)); } if (now >= _nextDailyArchive) { Rollup(ArchivePeriod.Daily); - _nextDailyArchive = _nextDailyArchive.AddDays(1); + AdvanceSchedule(ref _nextDailyArchive, now, static dt => dt.AddDays(1)); } if (now >= _nextMonthlyArchive) { Rollup(ArchivePeriod.Monthly); - _nextMonthlyArchive = _nextMonthlyArchive.AddMonths(1); + AdvanceSchedule(ref _nextMonthlyArchive, now, static dt => dt.AddMonths(1)); } Prune?.Invoke(); + } + catch (Exception ex) + { + logger.Error(ex, "Archive operation failed"); + } + finally + { _isArchiving = 0; - }, - date, - false - ); + } + } + ); + } + + /// + /// Forces an immediate rollup regardless of schedule. + /// + public static void ForceRollup() + { + if (Interlocked.CompareExchange(ref _isArchiving, 1, 0) == 1) + { + logger.Warning("Cannot force rollup — an archive operation is already in progress"); + return; } - private static void Rollup(ArchivePeriod archivePeriod) + ThreadPool.QueueUserWorkItem( + _ => + { + try + { + Rollup(ArchivePeriod.Hourly); + Rollup(ArchivePeriod.Daily); + Rollup(ArchivePeriod.Monthly); + Prune?.Invoke(); + } + catch (Exception ex) + { + logger.Error(ex, "Forced archive operation failed"); + } + finally + { + _isArchiving = 0; + } + } + ); + } + + /// + /// Advances a schedule marker past the current time to prevent redundant catchup cycles + /// after a long-running archive operation. + /// + private static void AdvanceSchedule(ref DateTime schedule, DateTime now, Func advance) + { + while (schedule <= now) { - if (!Directory.Exists(AutomaticBackupPath)) + schedule = advance(schedule); + } + } + + private static void Rollup(ArchivePeriod archivePeriod) + { + if (!Directory.Exists(AutomaticBackupPath)) + { + return; + } + + var currentRangeStart = Core.Now.ArchivePeriodStart(archivePeriod); + + var items = new Dictionary>(); + foreach (var path in Directory.EnumerateDirectories(AutomaticBackupPath)) + { + // Skip backup directories that are still being written (no completion marker) + if (!File.Exists(Path.Combine(path, BackupCompleteMarker))) { - return; + continue; } - var currentRangeStart = Core.Now.ArchivePeriodStart(archivePeriod); - var allFolders = Directory.EnumerateDirectories(AutomaticBackupPath); + var dirName = new DirectoryInfo(path).Name; - var items = new Dictionary>(); - foreach (var path in allFolders) + if (!TryGetDate(dirName, out var date)) { - var dirName = new DirectoryInfo(path).Name; + continue; + } - if (!TryGetDate(dirName, out var date)) - { - continue; - } + var rangeStart = date.ArchivePeriodStart(archivePeriod); - var rangeStart = date.ArchivePeriodStart(archivePeriod); + // Only archive the past + if (rangeStart >= currentRangeStart) + { + continue; + } - // Only archive the past - if (rangeStart >= currentRangeStart) + if (items.TryGetValue(rangeStart, out var value)) + { + value.Add(date, path); + } + else + { + value = new SortedDictionary(new DescendingComparer()) { - continue; - } + { date, path } + }; - if (items.TryGetValue(rangeStart, out var value)) + items.Add(rangeStart, value); + } + } + + var archivePeriodStr = archivePeriod.ToString(); + var archivePath = PathUtility.EnsureDirectory(Path.Combine(ArchivePath, archivePeriodStr)); + var archivePeriodStrLower = archivePeriodStr.ToLowerInvariant(); + + // Leave behind 1 hourly for daily, 1 daily for monthly. + var minimum = archivePeriod != ArchivePeriod.Monthly ? 1 : 0; + + foreach (var (rangeStart, sortedBackups) in items) + { + var backups = sortedBackups.Values; + if (backups.Count <= minimum) + { + continue; + } + + var stopWatch = Stopwatch.StartNew(); + + var fileName = $"{rangeStart.ToTimeStamp(archivePeriod)}.tar.zst"; + var archiveFilePath = Path.Combine(archivePath, fileName); + + // Journal: record start + var sourceList = new List(backups); + var journalEntry = ArchiveJournal.BeginOperation( + archivePeriod, rangeStart, archiveFilePath + ".tmp", archiveFilePath, sourceList + ); + + // Create archive using managed streaming compression + var entryCount = ManagedArchive.CreateTarZstd( + backups, archiveFilePath, AutomaticBackupPath, _compressionLevel + ); + + if (entryCount >= 0) + { + // Verify if enabled + if (_verifyArchives) { - value.Add(date, path); - } - else - { - value = new SortedDictionary(new DescendingComparer()) + var verifiedCount = ManagedArchive.CountEntries(archiveFilePath); + if (verifiedCount != entryCount) { - { date, path } - }; + logger.Warning( + "Archive verification failed for {Path}: expected {Expected} entries, got {Actual}", + archiveFilePath, entryCount, verifiedCount + ); + ArchiveJournal.RecordFailure(journalEntry, $"Verification failed: expected {entryCount}, got {verifiedCount}"); - items.Add(rangeStart, value); - } - } - - var archivePeriodStr = archivePeriod.ToString(); - var archivePath = PathUtility.EnsureDirectory(Path.Combine(ArchivePath, archivePeriodStr)); - var archivePeriodStrLower = archivePeriodStr.ToLowerInvariant(); - var extension = _compressionFormat.GetFileExtension(); - - // Leave behind 1 hourly for daily, 1 daily for monthly. - var minimum = archivePeriod != ArchivePeriod.Monthly ? 1 : 0; - - foreach (var (rangeStart, sortedBackups) in items) - { - var backups = sortedBackups.Values; - if (backups.Count <= minimum) - { - continue; + ArchiveFailed?.Invoke(new ArchiveFailedEventArgs( + archivePeriod, rangeStart, new InvalidOperationException("Archive verification failed") + )); + continue; + } } - var stopWatch = new Stopwatch(); - stopWatch.Start(); + var archiveSize = new FileInfo(archiveFilePath).Length; + ArchiveJournal.RecordArchived(journalEntry, entryCount, archiveSize); - var fileName = $"{rangeStart.ToTimeStamp(archivePeriod)}{extension}"; - var archiveFilePath = Path.Combine(archivePath, fileName); + stopWatch.Stop(); + logger.Information( + "Created {Period} archive at {Path} ({EntryCount} entries, {Size:F1} MB, {Elapsed:F2}s)", + archivePeriodStrLower, + archiveFilePath, + entryCount, + archiveSize / 1048576.0, + stopWatch.Elapsed.TotalSeconds + ); - var archiveCreated = CreateArchive(archiveFilePath, AutomaticBackupPath, backups); + // Distribute to registered destinations + var allDestinationsSucceeded = DistributeToDestinations( + archiveFilePath, archivePeriod, rangeStart, journalEntry + ); - if (archiveCreated) + // Only delete source backups if all destinations with retention succeeded + if (allDestinationsSucceeded) { - logger.Information( - "Created {Period} archive at {Path} ({Elapsed:F2} seconds)", - archivePeriodStrLower, - archiveFilePath, - stopWatch.Elapsed.TotalSeconds - ); - var i = minimum; foreach (var backup in backups) { @@ -346,120 +858,196 @@ namespace Server.Saves continue; } - Directory.Delete(backup, true); + RetryFileOperation(() => Directory.Delete(backup, true)); } + + ArchiveJournal.RecordCompleted(journalEntry); } else { - logger.Warning($"Failed to create {archivePeriodStrLower} archive"); + logger.Warning( + "Not pruning source backups for {Period} archive — some destinations failed", + archivePeriodStrLower + ); } + ArchiveCompleted?.Invoke(new ArchiveCompletedEventArgs( + archiveFilePath, archivePeriod, rangeStart, archiveSize, stopWatch.Elapsed.TotalSeconds + )); + } + else + { stopWatch.Stop(); + logger.Warning("Failed to create {Period} archive", archivePeriodStrLower); + ArchiveJournal.RecordFailure(journalEntry, "Archive creation failed"); + + ArchiveFailed?.Invoke(new ArchiveFailedEventArgs( + archivePeriod, rangeStart, new InvalidOperationException("Archive creation failed") + )); } } + } - private static bool CreateArchive(string archiveFilePath, string relativeTo, IEnumerable backups) => - _compressionFormat == CompressionFormat.Zstd - ? ZstdArchive.CreateFromPaths(backups, archiveFilePath, relativeTo) - : TarArchive.CreateFromPaths(backups, archiveFilePath, relativeTo); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static string ToTimeStamp(this DateTime date, ArchivePeriod archivePeriod) => - archivePeriod switch - { - ArchivePeriod.Monthly => date.ToString("yyyy-MM"), - ArchivePeriod.Daily => date.ToString("yyyy-MM-dd"), - ArchivePeriod.Hourly => date.ToString("yyyy-MM-dd-HH"), - _ => date.ToTimeStamp() - }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static DateTime ArchivePeriodStart(this DateTime date, ArchivePeriod archivePeriod) => - archivePeriod switch - { - ArchivePeriod.Monthly => date.Date.AddDays(-(date.Day - 1)), - ArchivePeriod.Daily => date.Date, - ArchivePeriod.Hourly => date.Date.AddHours(date.Hour), - _ => date - }; - - private static SortedDictionary.ValueCollection PathsByTimestampName(string path, bool files = false) + private static bool DistributeToDestinations( + string archiveFilePath, + ArchivePeriod period, + DateTime rangeStart, + ArchiveJournalEntry journalEntry) + { + var destinations = ArchiveDestinationRegistry.Destinations; + if (destinations.Count == 0) { - var allItems = files ? Directory.EnumerateFiles(path) : Directory.GetDirectories(path); - var items = new SortedDictionary(new DescendingComparer()); - foreach (var item in allItems) - { - string name; - if (files) - { - var fileName = new FileInfo(item).Name; - name = fileName[..fileName.IndexOfOrdinal('.')]; - } - else - { - name = new DirectoryInfo(item).Name; - } - - if (TryGetDate(name, out var date)) - { - // Might give wrong results if there is a file that matches: - // Example: 2021-09-01, and 2021-09-01-00 - items[date] = item; - } - } - - return items.Values; + ArchiveJournal.RecordDistributed(journalEntry, new Dictionary()); + return true; } - private static bool TryGetDate(string value, out DateTime date) - { - Span parts = stackalloc int[] { 0, 1, 1, 0, 0, 0 }; + var results = new Dictionary(destinations.Count); + var allSucceeded = true; + foreach (var dest in destinations) + { try { - var i = 0; - foreach (var part in value.Tokenize('-')) + var success = dest.SendArchive(archiveFilePath, period, rangeStart); + results[dest.Name] = success; + + if (success) { - if (!int.TryParse(part, out var partValue)) + logger.Debug("Archive sent to destination {Destination}", dest.Name); + } + else + { + logger.Warning("Destination {Destination} rejected archive", dest.Name); + if (dest.GetRetentionCount(period) > 0) { - break; + allSucceeded = false; } - - parts[i++] = partValue; } - - if (i == 0) - { - date = DateTime.MinValue; - return false; - } - - date = new DateTime( - parts[0], - parts[1], - parts[2], - parts[3], - parts[4], - parts[5], - DateTimeKind.Utc - ); - return true; } - catch + catch (Exception ex) + { + logger.Error(ex, "Failed to send archive to destination {Destination}", dest.Name); + results[dest.Name] = false; + if (dest.GetRetentionCount(period) > 0) + { + allSucceeded = false; + } + } + } + + ArchiveJournal.RecordDistributed(journalEntry, results); + return allSucceeded; + } + + private static void RetryFileOperation(Action operation) + { + for (var attempt = 0; attempt < _retryCount; attempt++) + { + try + { + operation(); + return; + } + catch (Exception ex) when (attempt < _retryCount - 1 && ex is IOException or UnauthorizedAccessException) + { + logger.Debug( + ex, + "File operation failed on attempt {Attempt}/{Max}, retrying", + attempt + 1, + _retryCount + ); + Thread.Sleep(_retryDelayMs * (attempt + 1)); + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static string ToTimeStamp(this DateTime date, ArchivePeriod archivePeriod) => + archivePeriod switch + { + ArchivePeriod.Monthly => date.ToString("yyyy-MM"), + ArchivePeriod.Daily => date.ToString("yyyy-MM-dd"), + ArchivePeriod.Hourly => date.ToString("yyyy-MM-dd-HH"), + _ => date.ToTimeStamp() + }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static DateTime ArchivePeriodStart(this DateTime date, ArchivePeriod archivePeriod) => + archivePeriod switch + { + ArchivePeriod.Monthly => date.Date.AddDays(-(date.Day - 1)), + ArchivePeriod.Daily => date.Date, + ArchivePeriod.Hourly => date.Date.AddHours(date.Hour), + _ => date + }; + + internal static SortedDictionary.ValueCollection PathsByTimestampName( + string path, bool files = false) + { + var allItems = files ? Directory.EnumerateFiles(path) : Directory.GetDirectories(path); + var items = new SortedDictionary(new DescendingComparer()); + foreach (var item in allItems) + { + string name; + if (files) + { + var fileName = new FileInfo(item).Name; + name = fileName[..fileName.IndexOfOrdinal('.')]; + } + else + { + name = new DirectoryInfo(item).Name; + } + + if (TryGetDate(name, out var date)) + { + // Might give wrong results if there is a file that matches: + // Example: 2021-09-01, and 2021-09-01-00 + items[date] = item; + } + } + + return items.Values; + } + + private static bool TryGetDate(string value, out DateTime date) + { + Span parts = [0, 1, 1, 0, 0, 0]; + + try + { + var i = 0; + foreach (var part in value.Tokenize('-')) + { + if (!int.TryParse(part, out var partValue)) + { + break; + } + + parts[i++] = partValue; + } + + if (i == 0) { date = DateTime.MinValue; return false; } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static string GetFileExtension(this CompressionFormat compressionFormat) => - compressionFormat switch - { - CompressionFormat.Zip => ".zip", - CompressionFormat.GZip => ".tar.gz", - CompressionFormat.Zstd => ".tar.zst", - _ => ".tar" - }; + date = new DateTime( + parts[0], + parts[1], + parts[2], + parts[3], + parts[4], + parts[5], + DateTimeKind.Utc + ); + return true; + } + catch + { + date = DateTime.MinValue; + return false; + } } } diff --git a/Projects/UOContent/World Saves/IArchiveDestination.cs b/Projects/UOContent/World Saves/IArchiveDestination.cs new file mode 100644 index 000000000..9cd3b0b37 --- /dev/null +++ b/Projects/UOContent/World Saves/IArchiveDestination.cs @@ -0,0 +1,46 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: IArchiveDestination.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; + +namespace Server.Saves; + +/// +/// Represents a destination for completed archive files. +/// Implementations run on a background thread (ThreadPool). +/// +public interface IArchiveDestination +{ + /// + /// A human-readable name for logging (e.g., "Local Filesystem", "S3 us-east-1"). + /// + string Name { get; } + + /// + /// Called after a local archive file is successfully created and verified. + /// Must be safe to call from a ThreadPool thread. + /// + /// Full path to the verified .tar.zst archive. + /// The archive period (Hourly, Daily, Monthly). + /// The start of the time range this archive covers. + /// True if the destination successfully received the archive. + bool SendArchive(string archiveFilePath, ArchivePeriod period, DateTime rangeStart); + + /// + /// Returns the number of local archives to retain for this period. + /// Destinations that keep their own copies may return 0. + /// + int GetRetentionCount(ArchivePeriod period); +} diff --git a/Projects/UOContent/World Saves/LocalArchiveDestination.cs b/Projects/UOContent/World Saves/LocalArchiveDestination.cs new file mode 100644 index 000000000..8348cde48 --- /dev/null +++ b/Projects/UOContent/World Saves/LocalArchiveDestination.cs @@ -0,0 +1,50 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: LocalArchiveDestination.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; + +namespace Server.Saves; + +/// +/// Built-in archive destination representing the local filesystem. +/// The archive is already on local disk, so SendArchive is a no-op. +/// Participates in the retention/pruning protocol via configurable counts. +/// +public class LocalArchiveDestination : IArchiveDestination +{ + public string Name => "Local Filesystem"; + + private readonly int _hourlyRetention; + private readonly int _dailyRetention; + private readonly int _monthlyRetention; + + public LocalArchiveDestination(int hourlyRetention, int dailyRetention, int monthlyRetention) + { + _hourlyRetention = hourlyRetention; + _dailyRetention = dailyRetention; + _monthlyRetention = monthlyRetention; + } + + // Archive is already on local filesystem — nothing to do. + public bool SendArchive(string archiveFilePath, ArchivePeriod period, DateTime rangeStart) => true; + + public int GetRetentionCount(ArchivePeriod period) => period switch + { + ArchivePeriod.Hourly => _hourlyRetention, + ArchivePeriod.Daily => _dailyRetention, + ArchivePeriod.Monthly => _monthlyRetention, + _ => 0 + }; +} diff --git a/Projects/UOContent/World Saves/SaveCommands.cs b/Projects/UOContent/World Saves/SaveCommands.cs index 29882bc4e..c0186b2b5 100644 --- a/Projects/UOContent/World Saves/SaveCommands.cs +++ b/Projects/UOContent/World Saves/SaveCommands.cs @@ -1,68 +1,104 @@ using Server.Saves; -namespace Server.Commands +namespace Server.Commands; + +public static class SaveCommands { - public static class SaveCommands + public static void Configure() { - public static void Configure() + CommandSystem.Register("Save", AccessLevel.Administrator, Save_OnCommand); + CommandSystem.Register("SetSaves", AccessLevel.Administrator, SetAutoSaves_OnCommand); + CommandSystem.Register("AutoSave", AccessLevel.Administrator, SetAutoSaves_OnCommand); + CommandSystem.Register("SaveFrequency", AccessLevel.Administrator, SetSaveFrequency_OnCommand); + CommandSystem.Register("PruneArchives", AccessLevel.Administrator, PruneArchives_OnCommand); + CommandSystem.Register("ArchiveStatus", AccessLevel.Administrator, ArchiveStatus_OnCommand); + CommandSystem.Register("ArchiveNow", AccessLevel.Administrator, ArchiveNow_OnCommand); + } + + [Usage("PruneArchives"), Description("Prunes archives folder.")] + private static void PruneArchives_OnCommand(CommandEventArgs e) => AutoArchive.PruneBackups(); + + [Usage("Save"), Description("Saves the world.")] + private static void Save_OnCommand(CommandEventArgs e) => AutoSave.Save(); + + [Usage("AutoSave "), Description("Enables or disables automatic shard saving.")] + public static void SetAutoSaves_OnCommand(CommandEventArgs e) + { + if (e.Length == 1) { - CommandSystem.Register("Save", AccessLevel.Administrator, Save_OnCommand); - CommandSystem.Register("SetSaves", AccessLevel.Administrator, SetAutoSaves_OnCommand); - CommandSystem.Register("AutoSave", AccessLevel.Administrator, SetAutoSaves_OnCommand); - CommandSystem.Register("SaveFrequency", AccessLevel.Administrator, SetSaveFrequency_OnCommand); - CommandSystem.Register("PruneArchives", AccessLevel.Administrator, PruneArchives_OnCommand); + var enabled = AutoSave.SavesEnabled = e.GetBoolean(0); + + e.Mobile.SendMessage(enabled ? "Saves have been enabled." : "Saves have been disabled."); } - - [Usage("PruneArchives"), Description("Prunes archives folder.")] - private static void PruneArchives_OnCommand(CommandEventArgs e) + else { - AutoArchive.PruneBackups(); - } - - [Usage("Save"), Description("Saves the world.")] - private static void Save_OnCommand(CommandEventArgs e) - { - AutoSave.Save(); - } - - [Usage("AutoSave "), Description("Enables or disables automatic shard saving.")] - public static void SetAutoSaves_OnCommand(CommandEventArgs e) - { - if (e.Length == 1) - { - - var enabled = AutoSave.SavesEnabled = e.GetBoolean(0); - - if (enabled) - { - e.Mobile.SendMessage("Saves have been enabled."); - } - else - { - e.Mobile.SendMessage("Saves have been disabled."); - } - } - else - { - e.Mobile.SendMessage("Format: AutoSave "); - } - } - - [Usage("SaveFrequency [warning duration]"), Description("Sets the save frequency starting at midnight local to the server.")] - public static void SetSaveFrequency_OnCommand(CommandEventArgs e) - { - if (e.Length < 1) - { - e.Mobile.SendMessage("Format: SaveFrequency [warning duration]"); - return; - } - - var saveDelay = e.GetTimeSpan(0); - var warningDelay = e.Length >= 2 ? e.GetTimeSpan(1) : AutoSave.Warning; - - AutoSave.ResetAutoSave(saveDelay, warningDelay); - - e.Mobile.SendMessage("Save frequency has been updated."); + e.Mobile.SendMessage("Format: AutoSave "); } } + + [Usage("SaveFrequency [warning duration]"), Description("Sets the save frequency starting at midnight local to the server.")] + public static void SetSaveFrequency_OnCommand(CommandEventArgs e) + { + if (e.Length < 1) + { + e.Mobile.SendMessage("Format: SaveFrequency [warning duration]"); + return; + } + + var saveDelay = e.GetTimeSpan(0); + var warningDelay = e.Length >= 2 ? e.GetTimeSpan(1) : AutoSave.Warning; + + AutoSave.ResetAutoSave(saveDelay, warningDelay); + + e.Mobile.SendMessage("Save frequency has been updated."); + } + + [Usage("ArchiveStatus")] + [Description("Shows archive system status, journal state, and registered destinations.")] + private static void ArchiveStatus_OnCommand(CommandEventArgs e) + { + var mobile = e.Mobile; + + mobile.SendMessage("--- Archive Status ---"); + mobile.SendMessage($"Next hourly: {AutoArchive.NextHourlyArchive:yyyy-MM-dd HH:mm}"); + mobile.SendMessage($"Next daily: {AutoArchive.NextDailyArchive:yyyy-MM-dd}"); + mobile.SendMessage($"Next monthly: {AutoArchive.NextMonthlyArchive:yyyy-MM}"); + + var destinations = ArchiveDestinationRegistry.Destinations; + mobile.SendMessage($"Registered destinations: {destinations.Count}"); + foreach (var dest in destinations) + { + mobile.SendMessage( + $" - {dest.Name} (retention: {dest.GetRetentionCount(ArchivePeriod.Hourly)}h/" + + $"{dest.GetRetentionCount(ArchivePeriod.Daily)}d/" + + $"{dest.GetRetentionCount(ArchivePeriod.Monthly)}m)" + ); + } + + var operations = ArchiveJournal.Operations; + var pending = 0; + var failed = 0; + for (var i = 0; i < operations.Count; i++) + { + var op = operations[i]; + if (op.State is ArchiveOperationState.Started or ArchiveOperationState.Archived + or ArchiveOperationState.Distributed) + { + pending++; + } + else if (op.State == ArchiveOperationState.Failed) + { + failed++; + } + } + + mobile.SendMessage($"Journal entries: {operations.Count} ({pending} pending, {failed} failed)"); + } + + [Usage("ArchiveNow"), Description("Forces an immediate archive rollup regardless of schedule.")] + private static void ArchiveNow_OnCommand(CommandEventArgs e) + { + e.Mobile.SendMessage("Forcing immediate archive rollup..."); + AutoArchive.ForceRollup(); + } }