From a55e271a69113722d72e2b9864086d1174d4f2fa Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Tue, 5 Oct 2021 08:54:44 -0700 Subject: [PATCH] fix: Cleans up file system paths and archiving (#815) * Makes EnsureDirectory properly work for relative and absolute paths * Adds a `PathUtility.GetFullPath` which returns full paths for relative paths to `Core.BaseDirectory`. If the path is absolute, it will return as-is. * Moves EnsureDirectory to `PathUtility`. So `ScriptsHandler.EnsureDirectory` and `AssemblyHandler.EnsureDirectory` are now `PathUtility.EnsureDirectory` * Fixes crash guard so that it copies accounts properly. * Changes world save and auto archive to use a random folder name inside of the temp folder. --- Projects/Server/AssemblyHandler.cs | 22 -- .../Serialization/GenericPersistence.cs | 4 +- Projects/Server/Text/StringHelpers.cs | 22 ++ Projects/Server/Timer/Timer.TimerWheel.cs | 2 +- Projects/Server/Utilities/PathUtility.cs | 67 ++++ Projects/Server/Utilities/Utility.cs | 22 -- Projects/Server/World/EntityPersistence.cs | 2 +- Projects/Server/World/World.cs | 21 +- .../UOContent.Tests/Fixtures/ServerFixture.cs | 3 +- .../Accounting/Account.Migrations.cs | 101 ++++++ Projects/UOContent/Accounting/Account.cs | 92 ----- Projects/UOContent/Compression/TarArchive.cs | 18 +- Projects/UOContent/Compression/ZstdArchive.cs | 14 +- .../Commands/ConvertPremiumSpawners.cs | 2 +- .../Commands/ExportSpawnersCommand.cs | 2 +- Projects/UOContent/Misc/CrashGuard.cs | 76 ++--- Projects/UOContent/World Saves/AutoArchive.cs | 318 +++++++++++------- 17 files changed, 447 insertions(+), 341 deletions(-) create mode 100644 Projects/Server/Utilities/PathUtility.cs create mode 100644 Projects/UOContent/Accounting/Account.Migrations.cs diff --git a/Projects/Server/AssemblyHandler.cs b/Projects/Server/AssemblyHandler.cs index 00e8d7e53..4cb69f497 100644 --- a/Projects/Server/AssemblyHandler.cs +++ b/Projects/Server/AssemblyHandler.cs @@ -173,28 +173,6 @@ namespace Server return null; } - - public static string EnsureDirectory(string dir) - { - var path = Path.Combine(Core.BaseDirectory, dir); - Directory.CreateDirectory(path); - - return path; - } - - public static void EnsureDirectory(FileInfo fi) - { - var dir = fi.DirectoryName; - if (dir != null) - { - Directory.CreateDirectory(dir); - } - } - - public static void EnsureDirectory(DirectoryInfo di) - { - Directory.CreateDirectory(di.FullName); - } } public class TypeCache diff --git a/Projects/Server/Serialization/GenericPersistence.cs b/Projects/Server/Serialization/GenericPersistence.cs index 2912664b7..eb878303c 100644 --- a/Projects/Server/Serialization/GenericPersistence.cs +++ b/Projects/Server/Serialization/GenericPersistence.cs @@ -41,7 +41,7 @@ namespace Server { var path = Path.Combine(savePath, name); - AssemblyHandler.EnsureDirectory(path); + PathUtility.EnsureDirectory(path); string binPath = Path.Combine(path, $"{name}.bin"); using var bin = new BinaryFileWriter(binPath, true); @@ -54,7 +54,7 @@ namespace Server { var path = Path.Combine(savePath, name); - AssemblyHandler.EnsureDirectory(path); + PathUtility.EnsureDirectory(path); string binPath = Path.Combine(path, $"{name}.bin"); diff --git a/Projects/Server/Text/StringHelpers.cs b/Projects/Server/Text/StringHelpers.cs index 7e05497c2..cf97b6a7c 100644 --- a/Projects/Server/Text/StringHelpers.cs +++ b/Projects/Server/Text/StringHelpers.cs @@ -169,6 +169,28 @@ namespace Server return str; } + public static string TrimMultiline(this string str, string lineSeparator = "\n") + { + var parts = str.Split(lineSeparator); + for (var i = 0; i < parts.Length; i++) + { + parts[i] = parts[i].Trim(); + } + + return string.Join(lineSeparator, parts); + } + + public static string IndentMultiline(this string str, string indent = "\t", string lineSeparator = "\n") + { + var parts = str.Split(lineSeparator); + for (var i = 0; i < parts.Length; i++) + { + parts[i] = $"{indent}{parts[i]}"; + } + + return string.Join(lineSeparator, parts); + } + public static List Wrap(this string value, int perLine, int maxLines) { if ((value = value?.Trim() ?? "").Length <= 0) diff --git a/Projects/Server/Timer/Timer.TimerWheel.cs b/Projects/Server/Timer/Timer.TimerWheel.cs index f8e4e451c..5e9d66334 100644 --- a/Projects/Server/Timer/Timer.TimerWheel.cs +++ b/Projects/Server/Timer/Timer.TimerWheel.cs @@ -242,7 +242,7 @@ namespace Server } #if DEBUG_TIMERS - tw.WriteLine("\nStack Traces:"); + tw.WriteLine($"{Environment.NewLine}Stack Traces:"); foreach (var kvp in DelayCallTimer._stackTraces) { tw.WriteLine(kvp.Value); diff --git a/Projects/Server/Utilities/PathUtility.cs b/Projects/Server/Utilities/PathUtility.cs new file mode 100644 index 000000000..d2206143d --- /dev/null +++ b/Projects/Server/Utilities/PathUtility.cs @@ -0,0 +1,67 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2021 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: PathUtility.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.IO; +using Server.Text; + +namespace Server +{ + public static class PathUtility + { + public static string EnsureDirectory(string dir) + { + var path = GetFullPath(dir, Core.BaseDirectory); + Directory.CreateDirectory(path); + + return path; + } + + public static void EnsureDirectory(this FileInfo fi) + { + var dir = GetFullPath(fi.DirectoryName, Core.BaseDirectory); + if (dir != null) + { + Directory.CreateDirectory(dir); + } + } + + public static void EnsureDirectory(this DirectoryInfo di) + { + var file = GetFullPath(di.FullName, Core.BaseDirectory); + Directory.CreateDirectory(file); + } + + public static string GetFullPath(string relativeOrAbsolutePath) => + GetFullPath(relativeOrAbsolutePath, Core.BaseDirectory); + + public static string GetFullPath(string relativeOrAbsolutePath, string basePath) => + relativeOrAbsolutePath switch + { + null => null, + "" => basePath, + _ => Path.IsPathRooted(relativeOrAbsolutePath) + ? relativeOrAbsolutePath + : Path.GetFullPath(relativeOrAbsolutePath, basePath) + }; + + public static string EnsureRandomPath(string basePath) + { + Span bytes = stackalloc byte[8]; + Utility.RandomBytes(bytes); + return EnsureDirectory(Path.Combine(basePath, bytes.ToHexString())); + } + } +} diff --git a/Projects/Server/Utilities/Utility.cs b/Projects/Server/Utilities/Utility.cs index 706fd09e5..286e488e7 100644 --- a/Projects/Server/Utilities/Utility.cs +++ b/Projects/Server/Utilities/Utility.cs @@ -1331,28 +1331,6 @@ namespace Server [MethodImpl(MethodImplOptions.AggressiveInlining)] public static T Max(T val, T max) where T : IComparable => val.CompareTo(max) > 0 ? val : max; - public static string TrimMultiline(this string str, string lineSeparator = "\n") - { - var parts = str.Split(lineSeparator); - for (var i = 0; i < parts.Length; i++) - { - parts[i] = parts[i].Trim(); - } - - return string.Join(lineSeparator, parts); - } - - public static string IndentMultiline(this string str, string indent = "\t", string lineSeparator = "\n") - { - var parts = str.Split(lineSeparator); - for (var i = 0; i < parts.Length; i++) - { - parts[i] = $"{indent}{parts[i]}"; - } - - return string.Join(lineSeparator, parts); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Tidy(this List list) where T : ISerializable { diff --git a/Projects/Server/World/EntityPersistence.cs b/Projects/Server/World/EntityPersistence.cs index 4ba5b68c2..bf2f415df 100644 --- a/Projects/Server/World/EntityPersistence.cs +++ b/Projects/Server/World/EntityPersistence.cs @@ -40,7 +40,7 @@ namespace Server var path = Path.Combine(savePath, typeName); - AssemblyHandler.EnsureDirectory(path); + PathUtility.EnsureDirectory(path); string idxPath = Path.Combine(path, $"{typeName}.idx"); string tdbPath = Path.Combine(path, $"{typeName}.tdb"); diff --git a/Projects/Server/World/World.cs b/Projects/Server/World/World.cs index 142b554a2..02b3013d5 100644 --- a/Projects/Server/World/World.cs +++ b/Projects/Server/World/World.cs @@ -46,7 +46,6 @@ namespace Server private static readonly ConcurrentQueue _decayQueue = new(); private static string _tempSavePath; // Path to the temporary folder for the save - private static string _savePath; // Path to "Saves" folder private static bool _enableSaveStats; public const bool DirtyTrackingEnabled = false; @@ -130,6 +129,8 @@ namespace Server internal static List MobileTypes { get; } = new(); internal static List GuildTypes { get; } = new(); + public static string SavePath { get; private set; } + public static WorldState WorldState { get; private set; } public static bool Saving => WorldState == WorldState.Saving; public static bool Running => WorldState is not WorldState.Loading and not WorldState.Initial; @@ -141,10 +142,12 @@ namespace Server public static void Configure() { - var tempSavePath = ServerConfiguration.GetOrUpdateSetting("world.tempSavePath", "temp"); - _tempSavePath = Path.Combine(Core.BaseDirectory, tempSavePath); + var tempSavePath = ServerConfiguration.GetSetting("world.tempSavePath", "temp"); + _tempSavePath = PathUtility.GetFullPath(tempSavePath); + var savePath = ServerConfiguration.GetOrUpdateSetting("world.savePath", "Saves"); - _savePath = Path.Combine(Core.BaseDirectory, savePath); + SavePath = PathUtility.GetFullPath(savePath); + _enableSaveStats = ServerConfiguration.GetOrUpdateSetting("world.enableSaveStats", false); // Mobiles & Items @@ -256,7 +259,7 @@ namespace Server logger.Information("Loading world"); var watch = Stopwatch.StartNew(); - Persistence.Load(_savePath); + Persistence.Load(SavePath); EventSink.InvokeWorldLoad(); ProcessSafetyQueues(); @@ -344,7 +347,7 @@ namespace Server var timestamp = Utility.GetTimeStamp(); var saveStatsPath = Path.Combine(Core.BaseDirectory, $"Logs/Saves/Save-Stats-{timestamp}.log"); - AssemblyHandler.EnsureDirectory(saveStatsPath); + PathUtility.EnsureDirectory(saveStatsPath); using var op = new StreamWriter(saveStatsPath, true); @@ -388,7 +391,7 @@ namespace Server { Exception exception = null; - var tempPath = Path.Combine(_tempSavePath, Utility.GetTimeStamp()); + var tempPath = PathUtility.EnsureRandomPath(_tempSavePath); try { @@ -417,8 +420,8 @@ namespace Server { try { - EventSink.InvokeWorldSavePostSnapshot(_savePath, tempPath); - Directory.Move(tempPath, _savePath); + EventSink.InvokeWorldSavePostSnapshot(SavePath, tempPath); + Directory.Move(tempPath, SavePath); } catch (Exception ex) { diff --git a/Projects/UOContent.Tests/Fixtures/ServerFixture.cs b/Projects/UOContent.Tests/Fixtures/ServerFixture.cs index ab391505b..a5cbdca8e 100644 --- a/Projects/UOContent.Tests/Fixtures/ServerFixture.cs +++ b/Projects/UOContent.Tests/Fixtures/ServerFixture.cs @@ -1,4 +1,5 @@ using System; +using System.Reflection; using Server.Misc; namespace Server.Tests @@ -8,8 +9,8 @@ namespace Server.Tests // Global setup static ServerFixture() { + Core.Assembly = Assembly.GetExecutingAssembly(); Core.LoopContext = new EventLoopContext(); - Core.Expansion = Expansion.EJ; // Load Configurations diff --git a/Projects/UOContent/Accounting/Account.Migrations.cs b/Projects/UOContent/Accounting/Account.Migrations.cs new file mode 100644 index 000000000..16b61efa1 --- /dev/null +++ b/Projects/UOContent/Accounting/Account.Migrations.cs @@ -0,0 +1,101 @@ +using System.Collections.Generic; +using System.IO; +using System.Net; +using Server.Accounting.Security; + +namespace Server.Accounting +{ + public partial class Account + { + private void MigrateFrom(V3Content content) + { + _username = content.Username; + _passwordAlgorithm = content.PasswordAlgorithm; + _password = content.Password; + _accessLevel = content.AccessLevel; + _flags = content.Flags; + Created = content.Created; + _lastLogin = content.LastLogin; + _totalGold = content.TotalGold; + _totalPlat = content.TotalPlat; + _mobiles = content.Mobiles; + _comments = content.Comments; + _tags = content.Tags; + _loginIPs = content.LoginIPs; + _ipRestrictions = content.IpRestrictions; + _totalGameTime = content.TotalGameTime; + _email = content.Email; + } + + private void Deserialize(IGenericReader reader, int version) + { + if (version != 2) + { + // Due to a bug where we were not versioning at all, reset so we don't have an issue deserializing + reader.Seek(0, SeekOrigin.Begin); + } + + _username = reader.ReadString(); + _passwordAlgorithm = version < 2 ? (PasswordProtectionAlgorithm)reader.ReadInt() : reader.ReadEnum(); + _password = reader.ReadString(); + _accessLevel = version < 2 ? (AccessLevel)reader.ReadInt() : reader.ReadEnum(); + _flags = reader.ReadInt(); + Created = reader.ReadDateTime(); + _lastLogin = reader.ReadDateTime(); + + _totalGold = reader.ReadInt(); + _totalPlat = reader.ReadInt(); + + var length = reader.ReadInt(); + _mobiles = new Mobile[length]; + for (int i = 0; i < length; i++) + { + _mobiles[i] = reader.ReadEntity(); + } + + length = reader.ReadInt(); + _comments = length > 0 ? new List(length) : null; + for (int i = 0; i < length; i++) + { + _comments!.Add(new AccountComment(reader)); + } + + length = reader.ReadInt(); + _tags = length > 0 ? new List(length) : null; + for (int i = 0; i < length; i++) + { + _tags!.Add(new AccountTag(reader)); + } + + length = reader.ReadInt(); + _loginIPs = new IPAddress[length]; + for (int i = 0; i < length; i++) + { + if (version < 2) + { + if (IPAddress.TryParse(reader.ReadString(), out var address)) + { + _loginIPs[i] = Utility.Intern(address); + } + } + else + { + _loginIPs[i] = reader.ReadIPAddress(); + } + } + + length = reader.ReadInt(); + _ipRestrictions = new string[length]; + for (int i = 0; i < length; i++) + { + _ipRestrictions[i] = reader.ReadString(); + } + + _totalGameTime = reader.ReadTimeSpan(); + + _email = reader.ReadString(); + + Timer.StartTimer(AfterDeserialization); + } + } +} diff --git a/Projects/UOContent/Accounting/Account.cs b/Projects/UOContent/Accounting/Account.cs index 5e2306f8d..43a1f8fb8 100644 --- a/Projects/UOContent/Accounting/Account.cs +++ b/Projects/UOContent/Accounting/Account.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.IO; using System.Net; using System.Xml; using Server.Accounting.Security; @@ -357,97 +356,6 @@ namespace Server.Accounting } } - private void MigrateFrom(V3Content content) - { - _username = content.Username; - _passwordAlgorithm = content.PasswordAlgorithm; - _password = content.Password; - _accessLevel = content.AccessLevel; - _flags = content.Flags; - Created = content.Created; - _lastLogin = content.LastLogin; - _totalGold = content.TotalGold; - _totalPlat = content.TotalPlat; - _mobiles = content.Mobiles; - _comments = content.Comments; - _tags = content.Tags; - _loginIPs = content.LoginIPs; - _ipRestrictions = content.IpRestrictions; - _totalGameTime = content.TotalGameTime; - _email = content.Email; - } - - private void Deserialize(IGenericReader reader, int version) - { - if (version != 2) - { - // Due to a bug where we were not versioning at all, reset so we don't have an issue deserializing - reader.Seek(0, SeekOrigin.Begin); - } - - _username = reader.ReadString(); - _passwordAlgorithm = version < 2 ? (PasswordProtectionAlgorithm)reader.ReadInt() : reader.ReadEnum(); - _password = reader.ReadString(); - _accessLevel = version < 2 ? (AccessLevel)reader.ReadInt() : reader.ReadEnum(); - _flags = reader.ReadInt(); - Created = reader.ReadDateTime(); - _lastLogin = reader.ReadDateTime(); - - _totalGold = reader.ReadInt(); - _totalPlat = reader.ReadInt(); - - var length = reader.ReadInt(); - _mobiles = new Mobile[length]; - for (int i = 0; i < length; i++) - { - _mobiles[i] = reader.ReadEntity(); - } - - length = reader.ReadInt(); - _comments = length > 0 ? new List(length) : null; - for (int i = 0; i < length; i++) - { - _comments!.Add(new AccountComment(reader)); - } - - length = reader.ReadInt(); - _tags = length > 0 ? new List(length) : null; - for (int i = 0; i < length; i++) - { - _tags!.Add(new AccountTag(reader)); - } - - length = reader.ReadInt(); - _loginIPs = new IPAddress[length]; - for (int i = 0; i < length; i++) - { - if (version < 2) - { - if (IPAddress.TryParse(reader.ReadString(), out var address)) - { - _loginIPs[i] = Utility.Intern(address); - } - } - else - { - _loginIPs[i] = reader.ReadIPAddress(); - } - } - - length = reader.ReadInt(); - _ipRestrictions = new string[length]; - for (int i = 0; i < length; i++) - { - _ipRestrictions[i] = reader.ReadString(); - } - - _totalGameTime = reader.ReadTimeSpan(); - - _email = reader.ReadString(); - - Timer.StartTimer(AfterDeserialization); - } - /// /// Deletes the account, all characters of the account, and all houses of those characters /// diff --git a/Projects/UOContent/Compression/TarArchive.cs b/Projects/UOContent/Compression/TarArchive.cs index 3bc024a52..0e5889e4a 100755 --- a/Projects/UOContent/Compression/TarArchive.cs +++ b/Projects/UOContent/Compression/TarArchive.cs @@ -50,8 +50,7 @@ namespace Server.Compression private static string DownloadTarForWindows() { - var tempDir = Path.Combine(Core.BaseDirectory, "temp"); - AssemblyHandler.EnsureDirectory("temp"); + var tempDir = PathUtility.EnsureRandomPath(Path.GetTempPath()); var libarchiveFile = Path.Combine(tempDir, "libarchive.zip"); using WebClient wc = new WebClient(); @@ -82,29 +81,28 @@ namespace Server.Compression } public static bool CreateFromPaths( - List paths, + IEnumerable paths, string destinationArchiveFileName, + string relativeTo, string compressCommand = null, string compressionProgramPath = null ) { _pathToTar ??= GetPathToTar(); - AssemblyHandler.EnsureDirectory(new FileInfo(destinationArchiveFileName)); - var di = new DirectoryInfo(paths[0]); - var directory = di.Parent!.FullName; + new FileInfo(destinationArchiveFileName).EnsureDirectory(); using var builder = new ValueStringBuilder(); - for (var i = 0; i < paths.Count; i++) + var i = 0; + foreach (var path in paths) { - var path = paths[i]; - builder.Append($"{(i > 0 ? " " : "")}\"{Path.GetRelativePath(directory, path)}\""); + builder.Append($"{(i++ > 0 ? " " : "")}\"{Path.GetRelativePath(relativeTo, path)}\""); } var pathsToCompress = builder.ToString(); var tarFlags = compressCommand == null ? "-acf" : "-cf"; var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : ""; - var arguments = $"{useExternalCompression}{tarFlags} \"{destinationArchiveFileName}\" -C \"{directory}\" {pathsToCompress}"; + var arguments = $"{useExternalCompression}{tarFlags} \"{destinationArchiveFileName}\" -C \"{relativeTo}\" {pathsToCompress}"; return RunTar(arguments, compressionProgramPath) == 0; } diff --git a/Projects/UOContent/Compression/ZstdArchive.cs b/Projects/UOContent/Compression/ZstdArchive.cs index 72cc56e94..b121e023a 100755 --- a/Projects/UOContent/Compression/ZstdArchive.cs +++ b/Projects/UOContent/Compression/ZstdArchive.cs @@ -42,17 +42,15 @@ namespace Server.Compression } } - return TarArchive.ExtractToDirectory(fileNamePath, outputDirectory, "zstd -d", _pathToZstd); + return TarArchive.ExtractToDirectory(fileNamePath, outputDirectory, "zstd -q -d", _pathToZstd); } public static bool CreateFromPaths( - List paths, + IEnumerable paths, string destinationArchiveFileName, - int compressionLevel = 10 + string relativeTo ) { - Debug.Assert(compressionLevel is >= 1 and <= 22, $"{nameof(compressionLevel)} must be between 1 and 22"); - // bsdtar has a bug and hangs, so we are doing it in two steps. if (Core.IsWindows) { @@ -60,7 +58,7 @@ namespace Server.Compression try { - if (!TarArchive.CreateFromPaths(paths, tempTarArchive)) + if (!TarArchive.CreateFromPaths(paths, relativeTo, tempTarArchive)) { return false; } @@ -70,7 +68,7 @@ namespace Server.Compression StartInfo = new ProcessStartInfo { FileName = Path.Combine(_pathToZstd, "zstd.exe"), - Arguments = $"-q -10 \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\"" + Arguments = $"-q \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\"" } }; @@ -89,7 +87,7 @@ namespace Server.Compression } } - return TarArchive.CreateFromPaths(paths, destinationArchiveFileName, "zstd -10", _pathToZstd); + return TarArchive.CreateFromPaths(paths, destinationArchiveFileName, relativeTo, "zstd -q", _pathToZstd); } } } diff --git a/Projects/UOContent/Engines/Spawners/Commands/ConvertPremiumSpawners.cs b/Projects/UOContent/Engines/Spawners/Commands/ConvertPremiumSpawners.cs index a542550bb..75f79c0e5 100644 --- a/Projects/UOContent/Engines/Spawners/Commands/ConvertPremiumSpawners.cs +++ b/Projects/UOContent/Engines/Spawners/Commands/ConvertPremiumSpawners.cs @@ -77,7 +77,7 @@ namespace Server.Engines.Spawners { var stemDir = stem[..^file.Name.Length]; var fullOutputDir = Path.Combine(outputDir, stemDir); - AssemblyHandler.EnsureDirectory(fullOutputDir); + PathUtility.EnsureDirectory(fullOutputDir); var outputFileName = $"{file.Name[..^file.Extension.Length]}.json"; ParsePremiumSpawnerFile(file.FullName, Path.Combine(fullOutputDir, outputFileName), options); diff --git a/Projects/UOContent/Engines/Spawners/Commands/ExportSpawnersCommand.cs b/Projects/UOContent/Engines/Spawners/Commands/ExportSpawnersCommand.cs index 467ea4fe3..4e24ef087 100644 --- a/Projects/UOContent/Engines/Spawners/Commands/ExportSpawnersCommand.cs +++ b/Projects/UOContent/Engines/Spawners/Commands/ExportSpawnersCommand.cs @@ -53,7 +53,7 @@ namespace Server.Engines.Spawners if (!Path.IsPathRooted(path)) { path = Path.Combine(Core.BaseDirectory, path); - AssemblyHandler.EnsureDirectory(directory); + PathUtility.EnsureDirectory(directory); } else if (!Directory.Exists(directory)) { diff --git a/Projects/UOContent/Misc/CrashGuard.cs b/Projects/UOContent/Misc/CrashGuard.cs index 7b029b35c..c41b213fa 100644 --- a/Projects/UOContent/Misc/CrashGuard.cs +++ b/Projects/UOContent/Misc/CrashGuard.cs @@ -4,6 +4,7 @@ using System.IO; using Server.Accounting; using Server.Logging; using Server.Network; +using Server.Saves; namespace Server.Misc { @@ -76,24 +77,23 @@ namespace Server.Misc } } - private static void CopyFile(string rootOrigin, string rootBackup, string path) + private static void DirectoryCopy(string sourceDirName, string destDirName) { - var originPath = Path.Combine(rootOrigin, path); - if (!File.Exists(originPath)) + // Get the subdirectories for the specified directory. + DirectoryInfo dir = new DirectoryInfo(sourceDirName); + DirectoryInfo[] dirs = dir.GetDirectories(); + + FileInfo[] files = dir.GetFiles(); + foreach (FileInfo file in files) { - return; + string tempPath = Path.Combine(destDirName, file.Name); + file.CopyTo(tempPath, false); } - var backupPath = Path.Combine(rootBackup, path); - Directory.CreateDirectory(Path.GetDirectoryName(backupPath)); - - try + foreach (DirectoryInfo subdir in dirs) { - File.Copy(originPath, backupPath); - } - catch - { - // ignored + string tempPath = Path.Combine(destDirName, subdir.Name); + DirectoryCopy(subdir.FullName, tempPath); } } @@ -105,26 +105,12 @@ namespace Server.Misc { var timeStamp = Utility.GetTimeStamp(); - var root = Core.BaseDirectory; - var rootBackup = Path.Combine(root, $"Backups/Crashed/{timeStamp}/"); - var rootOrigin = Path.Combine(root, "Saves/"); - - // Copy files - CopyFile(rootOrigin, rootBackup, "Accounts/Accounts.xml"); - - CopyFile(rootOrigin, rootBackup, "Items/Items.bin"); - CopyFile(rootOrigin, rootBackup, "Items/Items.idx"); - CopyFile(rootOrigin, rootBackup, "Items/Items.tdb"); - - CopyFile(rootOrigin, rootBackup, "Mobiles/Mobiles.bin"); - CopyFile(rootOrigin, rootBackup, "Mobiles/Mobiles.idx"); - CopyFile(rootOrigin, rootBackup, "Mobiles/Mobiles.tdb"); - - CopyFile(rootOrigin, rootBackup, "Guilds/Guilds.bin"); - CopyFile(rootOrigin, rootBackup, "Guilds/Guilds.idx"); - - CopyFile(rootOrigin, rootBackup, "Regions/Regions.bin"); - CopyFile(rootOrigin, rootBackup, "Regions/Regions.idx"); + var backupPath = PathUtility.EnsureDirectory(Path.Combine(AutoArchive.BackupPath, "Crashed", timeStamp)); + var savePath = World.SavePath; + if (Directory.Exists(savePath)) + { + DirectoryCopy(savePath, backupPath); + } logger.Information("Backup done"); } @@ -160,7 +146,7 @@ namespace Server.Misc try { - op.WriteLine("Mobiles: {0}", World.Mobiles.Count); + op.WriteLine($"Accounts: {Accounts.Count}"); } catch { @@ -169,15 +155,23 @@ namespace Server.Misc try { - op.WriteLine("Items: {0}", World.Items.Count); + op.WriteLine($"Mobiles: {World.Mobiles.Count}"); } catch { // ignored } - op.WriteLine("Exception:"); - op.WriteLine(e.Exception); + try + { + op.WriteLine($"Items: {World.Items.Count}"); + } + catch + { + // ignored + } + + op.WriteLine($"Exception: {e.Exception}"); op.WriteLine(); op.WriteLine("Clients:"); @@ -186,22 +180,22 @@ namespace Server.Misc { var states = TcpServer.Instances; - op.WriteLine("- Count: {0}", states.Count); + op.WriteLine($"- Count: {states.Count}"); foreach (var ns in TcpServer.Instances) { - op.Write("+ {0}:", ns); + op.Write($"+ {ns}:"); if (ns.Account is Account a) { - op.Write(" (account = {0})", a.Username); + op.Write($" (account = {a.Username})"); } var m = ns.Mobile; if (m != null) { - op.Write(" (mobile = 0x{0:X} '{1}')", m.Serial.Value, m.Name); + op.Write($" (mobile = 0x{m.Serial.Value:X} '{m.Name}')"); } op.WriteLine(); diff --git a/Projects/UOContent/World Saves/AutoArchive.cs b/Projects/UOContent/World Saves/AutoArchive.cs index 6e1149639..fc839e830 100755 --- a/Projects/UOContent/World Saves/AutoArchive.cs +++ b/Projects/UOContent/World Saves/AutoArchive.cs @@ -2,9 +2,9 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Runtime.CompilerServices; using System.Threading; +using Microsoft.Toolkit.HighPerformance; using Server.Compression; using Server.Logging; @@ -29,6 +29,7 @@ namespace Server.Saves { private static readonly ILogger logger = LogFactory.GetLogger(typeof(AutoArchive)); + private static string _tempArchivePath; private static int _isArchiving; private static DateTime _nextHourlyArchive; private static DateTime _nextDailyArchive; @@ -40,16 +41,25 @@ namespace Server.Saves public static Action Prune { get; set; } public static string ArchivePath { get; private set; } public static string BackupPath { get; private set; } + public static string AutomaticBackupPath { get; private set; } public static void Configure() { - BackupPath = ServerConfiguration.GetOrUpdateSetting("autoArchive.backupPath", "Backups/Automatic"); - ArchivePath = ServerConfiguration.GetOrUpdateSetting("autoArchive.archivePath", "Archives"); - _compressionFormat = ServerConfiguration.GetOrUpdateSetting("autoArchive.compressionFormat", CompressionFormat.Zstd); + var tempArchivePath = ServerConfiguration.GetSetting("autoArchive.tempArchivePath", "temp"); + _tempArchivePath = PathUtility.GetFullPath(tempArchivePath); + + var backupPath = ServerConfiguration.GetOrUpdateSetting("autoArchive.backupPath", "Backups"); + BackupPath = PathUtility.GetFullPath(backupPath); + AutomaticBackupPath = Path.Combine(BackupPath, "Automatic"); + + var archivePath = ServerConfiguration.GetOrUpdateSetting("autoArchive.archivePath", "Archives"); + ArchivePath = PathUtility.GetFullPath(archivePath); var useLocalArchives = ServerConfiguration.GetOrUpdateSetting("autoArchive.archiveLocally", true); _enablePruning = ServerConfiguration.GetOrUpdateSetting("autoArchive.enableArchivePruning", true); + _compressionFormat = ServerConfiguration.GetOrUpdateSetting("autoArchive.compressionFormat", CompressionFormat.Zstd); + if (useLocalArchives) { Archive = AutoArchiveLocally; @@ -85,8 +95,8 @@ namespace Server.Saves return; } - var backupPath = Path.Combine(BackupPath, Utility.GetTimeStamp()); - AssemblyHandler.EnsureDirectory(BackupPath); + Directory.CreateDirectory(AutomaticBackupPath); + var backupPath = Path.Combine(AutomaticBackupPath, Utility.GetTimeStamp()); Directory.Move(args.OldSavePath, backupPath); logger.Information($"Created backup at {backupPath}"); @@ -97,20 +107,16 @@ namespace Server.Saves private static void RestoreFromArchive() { var savePath = Path.Combine(Core.BaseDirectory, ServerConfiguration.GetSetting("world.savePath", "Saves")); - - var files = Directory.Exists(savePath) ? Directory.GetFiles(savePath, "????-??-??-??-??-??.*") : null; - - if (files == null || files.Length == 0) + if (!Directory.Exists(savePath)) { return; } - var latestFiles = GetInRange(files, DateTime.MinValue, DateTime.MaxValue, true); - - foreach (var file in latestFiles) + foreach (var file in PathsByTimestampName(savePath, true)) { if (RestoreFromFile(file, savePath)) { + File.Delete(file); break; } } @@ -128,19 +134,10 @@ namespace Server.Saves logger.Information($"Restoring latest world save from archive {fileName}"); - var tempFolder = Path.Combine(Core.BaseDirectory, "temp"); - AssemblyHandler.EnsureDirectory(tempFolder); - bool successful; - - if (fileName.EndsWithOrdinal(".tar.zst")) - { - successful = ZstdArchive.ExtractToDirectory(fi.FullName, tempFolder); - } - else - { - TarArchive.ExtractToDirectory(fi.FullName, tempFolder); - successful = true; - } + var tempPath = PathUtility.EnsureRandomPath(_tempArchivePath); + var successful = fileName.EndsWithOrdinal(".tar.zst") + ? ZstdArchive.ExtractToDirectory(fi.FullName, tempPath) + : TarArchive.ExtractToDirectory(fi.FullName, tempPath); if (!successful) { @@ -148,25 +145,17 @@ namespace Server.Saves return false; } - var allFolders = Directory.EnumerateDirectories(tempFolder, "????-??-??-??-??-??"); - var worldSaveFolders = GetInRange(allFolders, DateTime.MinValue, DateTime.MaxValue); - var first = true; - foreach (var folder in worldSaveFolders) + foreach (var folder in PathsByTimestampName(tempPath)) { - if (first) - { - first = false; - Directory.Delete(savePath, true); - var dirInfo = new DirectoryInfo(folder); - logger.Information($"Restoring backup {dirInfo.Name}"); - Directory.Move(folder, savePath); - } - else - { - Directory.Delete(folder, true); - } + Directory.Delete(savePath, true); + var dirInfo = new DirectoryInfo(folder); + logger.Information($"Restoring backup {dirInfo.Name}"); + Directory.Move(folder, savePath); + break; } + Directory.Delete(tempPath, true); + return true; } @@ -180,14 +169,26 @@ namespace Server.Saves PruneLocalArchives(ArchivePeriod.Monthly, 12); } - if (Directory.Exists(BackupPath)) + if (!Directory.Exists(AutomaticBackupPath)) { - var allFolders = Directory.EnumerateDirectories(BackupPath, "????-??-??-??-??-??", SearchOption.AllDirectories); - var latestBackupFolders = GetInRange(allFolders, DateTime.MinValue, Core.Now.AddMonths(-1)); + return; + } - foreach (var folder in latestBackupFolders) + var allFolders = Directory.EnumerateDirectories(AutomaticBackupPath); + var threshold = Core.Now.AddMonths(-1); + + foreach (var folder in allFolders) + { + var dirName = new DirectoryInfo(folder).Name; + + if (!TryGetDate(dirName, out var date)) { - logger.Information($"Pruning backup {folder}"); + continue; + } + + if (date < threshold) + { + logger.Information($"Pruning old backup {folder}"); Directory.Delete(folder, true); } } @@ -196,12 +197,14 @@ namespace Server.Saves private static void PruneLocalArchives(ArchivePeriod period, int minRetained) { var periodStr = period.ToString(); - var path = Path.Combine(ArchivePath, periodStr); - var allFiles = Directory.EnumerateFiles(path, "????-??-??-??-??-??.*"); - var archives = GetInRange(allFiles, DateTime.MinValue, DateTime.MaxValue, true); + var archivePath = Path.Combine(ArchivePath, periodStr); + if (!Directory.Exists(archivePath)) + { + return; + } var periodLowerStr = periodStr.ToLowerInvariant(); - foreach (var archive in archives) + foreach (var archive in PathsByTimestampName(archivePath, true)) { if (minRetained > 0) { @@ -234,98 +237,150 @@ namespace Server.Saves { if (date >= _nextHourlyArchive) { - var lastHour = date.Date.AddHours(date.Hour); - Rollup(ArchivePeriod.Hourly, lastHour.AddHours(-1), lastHour.ToTimeStamp()); - _nextHourlyArchive = _nextHourlyArchive.AddHours(2); + Rollup(ArchivePeriod.Hourly); + _nextHourlyArchive = _nextHourlyArchive.AddHours(1); } if (date >= _nextDailyArchive) { - var yesterday = date.Date.AddDays(-1); - Rollup(ArchivePeriod.Daily, yesterday, yesterday.ToTimeStamp()); - _nextDailyArchive = _nextDailyArchive.AddDays(2); + Rollup(ArchivePeriod.Daily); + _nextDailyArchive = _nextDailyArchive.AddDays(1); } if (date >= _nextMonthlyArchive) { - var lastMonth = new DateTime(date.Year, date.Month, 1).AddMonths(-1); - Rollup(ArchivePeriod.Monthly, lastMonth, lastMonth.ToTimeStamp()); - _nextMonthlyArchive = _nextMonthlyArchive.AddMonths(2); + Rollup(ArchivePeriod.Monthly); + _nextMonthlyArchive = _nextMonthlyArchive.AddMonths(1); } Prune?.Invoke(); - _isArchiving = 0; }, null ); } - private static void Rollup(ArchivePeriod archivePeriod, DateTime rangeStart, string archiveNameNoExtension) + private static void Rollup(ArchivePeriod archivePeriod) { - if (!Directory.Exists(BackupPath)) + if (!Directory.Exists(AutomaticBackupPath)) { return; } + var currentRangeStart = Core.Now.ArchivePeriodStart(archivePeriod); + var allFolders = Directory.EnumerateDirectories(AutomaticBackupPath); + + var items = new Dictionary>(); + foreach (var path in allFolders) + { + var dirName = new DirectoryInfo(path).Name; + + if (!TryGetDate(dirName, out var date)) + { + continue; + } + + var rangeStart = date.ArchivePeriodStart(archivePeriod); + + // Only archive the past + if (rangeStart >= currentRangeStart) + { + continue; + } + + if (items.TryGetValue(rangeStart, out var value)) + { + value.Add(date, path); + } + else + { + value = new SortedDictionary(new DescendingComparer()) + { + { date, path } + }; + + items.Add(rangeStart, value); + } + } + var archivePeriodStr = archivePeriod.ToString(); + var archivePath = PathUtility.EnsureDirectory(Path.Combine(ArchivePath, archivePeriodStr)); var archivePeriodStrLower = archivePeriodStr.ToLowerInvariant(); - - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var allFolders = Directory.EnumerateDirectories(BackupPath, "????-??-??-??-??-??"); - - var rangeEnd = archivePeriod switch - { - ArchivePeriod.Monthly => rangeStart.AddMonths(1), - ArchivePeriod.Daily => rangeStart.AddDays(1), - _ => rangeStart.AddHours(1), - }; - - var latestFolders = GetInRange(allFolders, rangeStart, rangeEnd).ToList(); - - // We don't want to re-archive a single save. This usually means we rebooted and found the one we purposefully left behind - if (latestFolders.Count <= 1) - { - return; - } - - var archivePath = Path.Combine(ArchivePath, archivePeriodStr); - AssemblyHandler.EnsureDirectory(archivePath); - var extension = _compressionFormat.GetFileExtension(); - var archiveFilePath = Path.Combine(archivePath, $"{archiveNameNoExtension}{extension}"); - logger.Information($"Creating {archivePeriodStrLower} archive"); - var archiveCreated = _compressionFormat == CompressionFormat.Zstd - ? ZstdArchive.CreateFromPaths(latestFolders, archiveFilePath) - : TarArchive.CreateFromPaths(latestFolders, archiveFilePath); + // Leave behind 1 hourly for daily, 1 daily for monthly. + var minimum = archivePeriod != ArchivePeriod.Monthly ? 1 : 0; - if (archiveCreated) + foreach (var (rangeStart, sortedBackups) in items) { - stopWatch.Stop(); - var elapsed = stopWatch.Elapsed.TotalSeconds; - logger.Information($"Created {archivePeriodStrLower} archive at {archiveFilePath} ({elapsed:F2} seconds)"); - - // Keep the latest one, but delete the rest. - for (var i = 1; i < latestFolders.Count; i++) + var backups = sortedBackups.Values; + if (backups.Count == 0) { - Directory.Delete(latestFolders[i], true); + continue; } - } - else - { - logger.Warning($"Failed to create {archivePeriodStrLower} archive"); + + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + var fileName = $"{rangeStart.ToTimeStamp(archivePeriod)}{extension}"; + var archiveFilePath = Path.Combine(archivePath, fileName); + + var archiveCreated = CreateArchive(archiveFilePath, AutomaticBackupPath, backups); + + if (archiveCreated) + { + var elapsed = stopWatch.Elapsed.TotalSeconds; + logger.Information($"Created {archivePeriodStrLower} archive at {archiveFilePath} ({elapsed:F2} seconds)"); + + var i = minimum; + foreach (var backup in backups) + { + // Keep the latest one, but delete the rest. + if (i-- > 0) + { + continue; + } + + Directory.Delete(backup, true); + } + } + else + { + logger.Warning($"Failed to create {archivePeriodStrLower} archive"); + } + + stopWatch.Stop(); } } - private static IEnumerable GetInRange( - IEnumerable allItems, - DateTime rangeStart, - DateTime rangeEnd, - bool files = false - ) + private static bool CreateArchive(string archiveFilePath, string relativeTo, IEnumerable backups) => + _compressionFormat == CompressionFormat.Zstd + ? ZstdArchive.CreateFromPaths(backups, archiveFilePath, relativeTo) + : TarArchive.CreateFromPaths(backups, archiveFilePath, relativeTo); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static string ToTimeStamp(this DateTime date, ArchivePeriod archivePeriod) => + archivePeriod switch + { + ArchivePeriod.Monthly => date.ToString("yyyy-MM"), + ArchivePeriod.Daily => date.ToString("yyyy-MM-dd"), + ArchivePeriod.Hourly => date.ToString("yyyy-MM-dd-HH"), + _ => date.ToTimeStamp() + }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static DateTime ArchivePeriodStart(this DateTime date, ArchivePeriod archivePeriod) => + archivePeriod switch + { + ArchivePeriod.Monthly => date.Date.AddDays(-(date.Day - 1)), + ArchivePeriod.Daily => date.Date, + ArchivePeriod.Hourly => date.Date.AddHours(date.Hour), + _ => date + }; + + private static IEnumerable PathsByTimestampName(string path, bool files = false) { + var allItems = files ? Directory.GetFiles(path) : Directory.GetDirectories(path); var items = new SortedDictionary(new DescendingComparer()); foreach (var item in allItems) { @@ -333,14 +388,14 @@ namespace Server.Saves if (files) { var fileName = new FileInfo(item).Name; - name = fileName[..fileName.IndexOfOrdinal(".")]; + name = fileName[..fileName.IndexOf('.')]; } else { name = new DirectoryInfo(item).Name; } - if (IsInRange(name, rangeStart, rangeEnd, out var date)) + if (TryGetDate(name, out var date)) { items.Add(date, item); } @@ -349,38 +404,41 @@ namespace Server.Saves return items.Values; } - private static bool IsInRange(string name, DateTime start, DateTime end, out DateTime date) => - TryGetDate(name, out date) && start <= date && end >= date; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool TryGetDate(string value, out DateTime date) { - // Expecting YYYY-MM-DD-HH-mm-ss - var split = value.Split("-"); - if (split.Length != 6) - { - date = DateTime.MinValue; - return false; - } + Span parts = stackalloc int[] { 0, 1, 1, 0, 0, 0 }; try { + var i = 0; + foreach (var part in value.Tokenize('-')) + { + parts[i++] = int.Parse(part); + } + + if (i == 0) + { + date = DateTime.MinValue; + return false; + } + date = new DateTime( - int.Parse(split[0]), - int.Parse(split[1]), - int.Parse(split[2]), - int.Parse(split[3]), - int.Parse(split[4]), - int.Parse(split[5]) + parts[0], + parts[1], + parts[2], + parts[3], + parts[4], + parts[5], + DateTimeKind.Utc ); + return true; } catch { + logger.Warning($"Path was not in the correct date format: {value}"); date = DateTime.MinValue; return false; } - - return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)]