feat: Adds auto archiving (#794)

## Adds Auto Archiving
Backups are archived once an hour, day, and month. Archives older than 60 days are pruned automatically.

_Note: Automatic pruning is off by default_

### Archive compression format
The following formats are supported:
* Zstd (The fastest with best compression ratio)
* GZip
* Zip
* None (Tar)

_Note: By default archives use [zstandard](http://facebook.github.io/zstd/) format.
The archive format can be changed in modernuo.json `autoArchive.compressionFormat`_

### Restoring world from archive
Move the archive to the Saves folder (tar.zst file). On startup the server will extract the file and restore the latest save. See pictures below.

### Manually extracting an archives
#### Windows
* Use the latest version of [7-zip w/ ZStandard](https://github.com/mcmilk/7-Zip-zstd/releases/latest)
  1. Extract the `.tar.zst` file.
  2. Extract the `.tar` file. (Yes you have to do it in two steps)
* On Windows 10 you can use the command line. `zstd.exe` is in the Assemblies folder after building ModernUO.
  1. `tar --use-compress-program "Distribution\Assemblies\zstd.exe -d" -xvf "Archives\Hourly\archivefile.tar.zst" -C "path to where you want to extract it"`
#### Mac
* Install [Keka](https://www.keka.io)
  1. Drop the .tar.zst onto the keka interface.
#### Linux
  1. Install zstd from a package manager
  2. Run `tar -I zstd -xvf "Archives\Hourly\archivefile.tar.zst" -C "path to where you want to extract it"`

### Other changes
* Changes Autosave to occur at the same time no matter when the server is booted.
* Adds `[SaveFrequency <delay> [warning]`command to set save frequency and warning frequency in-game.
* Adds support for time zones that are configurable. The system timezone can also be manually configured. Check `TimeZoneHandler.cs` for details.

<img width="257" alt="Screen Shot 2021-09-22 at 11 23 18 PM" src="https://user-images.githubusercontent.com/3953314/134464929-a5bf3cd8-2ef0-4476-a9ad-71d0816a19ac.png">
<img width="241" alt="Screen Shot 2021-09-22 at 11 23 39 PM" src="https://user-images.githubusercontent.com/3953314/134464945-5f6f96dc-3d1e-434d-8256-5c6b3705e786.png">
<img width="836" alt="Screen Shot 2021-09-22 at 11 34 39 PM" src="https://user-images.githubusercontent.com/3953314/134464957-625e57f2-ef47-4ca1-a0a7-cd40c9b6539c.png">
<img width="631" alt="Screen Shot 2021-09-22 at 11 34 50 PM" src="https://user-images.githubusercontent.com/3953314/134464969-b2d65c0d-f8f5-497a-a832-5d805e8e0b22.png">

.gitignore

@@ -2,13 +2,16 @@
 /Distribution/ModernUO
 /Distribution/ModernUO.*
 /Distribution/Assemblies
+/Distribution/bsdtar
 /Distribution/Configuration/modernuo.json
 /Distribution/Configuration/email-settings.json
 /Distribution/Configuration/throttles.json
 /Distribution/Logs
+/Distribution/Archives
 /Distribution/Backups
 /Distribution/Saves
 /Distribution/docs
+/Distribution/temp
 /Distribution/*.dylib
 /Distribution/*.so
 /Distribution/*.dll

Projects/Server/AssemblyHandler.cs

@@ -181,6 +181,20 @@ namespace Server
 
             return path;
         }
+
+        public static void EnsureDirectory(FileInfo fi)
+        {
+            var dir = fi.DirectoryName;
+            if (dir != null)
+            {
+                Directory.CreateDirectory(dir);
+            }
+        }
+
+        public static void EnsureDirectory(DirectoryInfo di)
+        {
+            Directory.CreateDirectory(di.FullName);
+        }
     }
 
     public class TypeCache

Projects/Server/Comparers/DescendingComparer.cs

@@ -0,0 +1,24 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2021 - ModernUO Development Team                       *
+ * Email: hi@modernuo.com                                                *
+ * File: DescendingComparer.cs                                           *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+using System;
+using System.Collections.Generic;
+
+namespace Server
+{
+    public class DescendingComparer<T> : IComparer<T> where T : IComparable<T> {
+        public int Compare(T x, T y) => y?.CompareTo(x) ?? 1;
+    }
+}

Projects/Server/Configuration/ServerConfiguration.cs

@@ -169,6 +169,17 @@ namespace Server
             return value;
         }
 
+        public static void SetSetting(string key, TimeSpan value) => SetSetting(key, value.ToString());
+
+        public static void SetSetting(string key, int value) => SetSetting(key, value.ToString());
+
+        public static void SetSetting(string key, long value) => SetSetting(key, value.ToString());
+
+        public static void SetSetting(string key, bool value) => SetSetting(key, value.ToString());
+
+        public static void SetSetting<T>(string key, T value) where T : struct, Enum =>
+            SetSetting(key, value.ToString());
+
         public static void SetSetting(string key, string value)
         {
             m_Settings.Settings[key] = value;

Projects/Server/Json/Converters/GuidConverter.cs

@@ -14,7 +14,6 @@
  *************************************************************************/
 
 using System;
-using System.Net;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 

Projects/Server/Json/Converters/GuidConverterFactory.cs

@@ -14,7 +14,6 @@
  *************************************************************************/
 
 using System;
-using System.Net;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 

Projects/Server/Main.cs

@@ -494,7 +494,7 @@ namespace Server
             try
             {
 #if DEBUG
-                var idleCPU = true;
+                const bool idleCPU = true;
 #else
                 var idleCPU = ServerConfiguration.GetOrUpdateSetting("core.enableIdleCPU", false);
 #endif

Projects/Server/TimeZones/TimeZoneHandler.cs

@@ -0,0 +1,80 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2021 - ModernUO Development Team                       *
+ * Email: hi@modernuo.com                                                *
+ * File: TimeZoneHandler.cs                                              *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.CompilerServices;
+using Server.Json;
+
+namespace Server
+{
+    /**
+     * TimeZoneHandler provides a cross platform compatible way to handle system timezones.
+     *
+     * By default the system timezone is used.
+     * To manually configure the local timezone add the following setting to modernuo.json:
+     * "system.localTimeZone": "Pacific Standard Time"
+     * All valid timezones can be found in the Distribution/Data/timezones.json file.
+     *
+     * Notes for timezones.json:
+     * By default, Windows will use the values in the "timezone" property.
+     * By default, Unix/MacOS will use the values in the "region" property.
+     * system.localTimeZone can be either timezone or a region.
+     *
+     * Example:
+     * "system.localTimeZone": "Europe/Lisbon"
+     */
+    public static class TimeZoneHandler
+    {
+        private static readonly Dictionary<string, TimeZoneInfo> _timeZoneById = new();
+        public static TimeZoneInfo SystemTimeZone { get; private set; }
+
+        static TimeZoneHandler()
+        {
+            var timezones = JsonConfig.Deserialize<TimeZoneWithRegions[]>(Path.Combine(Core.BaseDirectory, "Data/timezones.json"));
+            foreach (var tz in timezones)
+            {
+                // Get the timezone if we are on Windows
+                var tzInfo = Core.IsWindows ? TimeZoneInfo.FindSystemTimeZoneById(tz.TimeZone) : null;
+
+                foreach (var region in tz.Regions)
+                {
+                    // Get the timezone by region if we are on linux
+                    tzInfo ??= TimeZoneInfo.FindSystemTimeZoneById(region);
+                    _timeZoneById[region] = tzInfo;
+                }
+
+                _timeZoneById[tz.TimeZone] = tzInfo;
+            }
+        }
+
+        public static void Configure()
+        {
+            var tzId = ServerConfiguration.GetSetting("system.localTimeZone", TimeZoneInfo.Local.Id);
+            SystemTimeZone = FindTimeZoneById(tzId);
+        }
+
+        /**
+         * Cross platform version of TimeZoneInfo.FindSystemTimeZoneById
+         */
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static TimeZoneInfo FindTimeZoneById(string id) => _timeZoneById[id];
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static DateTime ToSystemLocalTime(this DateTime date) =>
+            TimeZoneInfo.ConvertTimeFromUtc(date, SystemTimeZone);
+    }
+}

Projects/Server/TimeZones/TimeZoneWithRegions.cs

@@ -0,0 +1,28 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2021 - ModernUO Development Team                       *
+ * Email: hi@modernuo.com                                                *
+ * File: TimeZoneWithRegions.cs                                          *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+using System.Text.Json.Serialization;
+
+namespace Server
+{
+    public record TimeZoneWithRegions
+    {
+        [JsonPropertyName("timezone")]
+        public string TimeZone { get; init; }
+
+        [JsonPropertyName("regions")]
+        public string[] Regions { get; init; }
+    }
+}

Projects/Server/Timer/Timer.cs

@@ -35,9 +35,9 @@ namespace Server
         private Timer _nextTimer;
         private Timer _prevTimer;
 
-        public Timer(TimeSpan delay) : this(delay, TimeSpan.Zero, 1)
+        public Timer(TimeSpan delay) => Init(delay, TimeSpan.Zero, 1);
-        {
+
-        }
+        public Timer(TimeSpan interval, int count) => Init(interval, interval, count);
 
         public Timer(TimeSpan delay, TimeSpan interval, int count = 0) => Init(delay, interval, count);
 

Projects/Server/Utilities/Utility.cs

@@ -927,14 +927,11 @@ namespace Server
             return outputList;
         }
 
-        public static bool ToBoolean(string value)
+        public static bool ToBoolean(string value) =>
-        {
+            bool.TryParse(value, out var b) && b ||
-#pragma warning disable CA1806 // Do not ignore method results
+            value.InsensitiveEquals("enabled") ||
-            bool.TryParse(value, out var b);
+            value.InsensitiveEquals("on") ||
-#pragma warning restore CA1806 // Do not ignore method results
+            !value.InsensitiveEquals("disabled") && !value.InsensitiveEquals("off");
-
-            return b;
-        }
 
         public static double ToDouble(string value)
         {
@@ -1484,7 +1481,10 @@ namespace Server
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static string GetTimeStamp() => Core.Now.ToString("yyyy-MM-dd-HH-mm-ss");
+        public static string GetTimeStamp() => Core.Now.ToTimeStamp();
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static string ToTimeStamp(this DateTime dt) => dt.ToString("yyyy-MM-dd-HH-mm-ss");
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static void Add<T>(ref List<T> list, T value)
@@ -1639,5 +1639,26 @@ namespace Server
             dict.Clear();
             dict = null;
         }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void DateToComponents(
+            DateTime date,
+            out int year,
+            out int month,
+            out int day,
+            out DayOfWeek dayOfWeek,
+            out int hour,
+            out int min,
+            out int sec
+        )
+        {
+            year = date.Year;
+            month = date.Month;
+            day = date.Day;
+            dayOfWeek = date.DayOfWeek;
+            hour = date.Hour;
+            min = date.Minute;
+            sec = date.Second;
+        }
     }
 }

Projects/Server/World/World.cs

@@ -131,7 +131,7 @@ namespace Server
 
         public static WorldState WorldState { get; private set; }
         public static bool Saving => WorldState == WorldState.Saving;
-        public static bool Running => WorldState != WorldState.Loading && WorldState != WorldState.Initial;
+        public static bool Running => WorldState == WorldState.Running;
         public static bool Loading => WorldState == WorldState.Loading;
 
         public static Dictionary<Serial, Mobile> Mobiles { get; private set; }

Projects/UOContent/Commands/Handlers.cs

@@ -11,6 +11,7 @@ using Server.Misc;
 using Server.Mobiles;
 using Server.Multis;
 using Server.Network;
+using Server.Saves;
 using Server.Spells;
 using Server.Targeting;
 using Server.Targets;
@@ -44,8 +45,6 @@ namespace Server.Commands
 
             Register("Help", AccessLevel.Player, Help_OnCommand);
 
-            Register("Save", AccessLevel.Administrator, Save_OnCommand);
-
             Register("Move", AccessLevel.GameMaster, Move_OnCommand);
             Register("Client", AccessLevel.Counselor, Client_OnCommand);
 
@@ -459,12 +458,6 @@ namespace Server.Commands
             e.Mobile.Target = new PickMoveTarget();
         }
 
-        [Usage("Save"), Description("Saves the world.")]
-        private static void Save_OnCommand(CommandEventArgs e)
-        {
-            AutoSave.Save();
-        }
-
         private static bool FixMap(ref Map map, ref Point3D loc, Item item) =>
             map != null && map != Map.Internal || item.RootParent is Mobile m &&
             FixMap(ref map, ref loc, m);

Projects/UOContent/Compression/TarArchive.cs

@@ -0,0 +1,113 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
+using System.Net;
+using Server.Buffers;
+
+namespace Server.Compression
+{
+    public static class TarArchive
+    {
+        private const string _libArchiveWindowsUrl = @"https://www.libarchive.org/downloads/libarchive-v3.5.2-win64.zip";
+        private static string _pathToTar;
+
+        private static string GetPathToTar()
+        {
+            if (!Core.IsWindows || File.Exists(@"C:\Windows\system32\tar.exe"))
+            {
+                return "tar";
+            }
+
+            var tarPath = Path.Combine(Core.BaseDirectory, "bsdtar/bsdtar.exe");
+            return File.Exists(tarPath) ? tarPath : DownloadTarForWindows();
+        }
+
+        private static int RunTar(string arguments, string compressionProgramPath)
+        {
+            var process = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    FileName = _pathToTar,
+                    Arguments = arguments
+                }
+            };
+
+            if (compressionProgramPath != null)
+            {
+                var envPaths = $"{process.StartInfo.EnvironmentVariables["PATH"]}:{compressionProgramPath}";
+                process.StartInfo.EnvironmentVariables["PATH"] = envPaths;
+            }
+
+            process.Start();
+            process.WaitForExit();
+
+            return process.ExitCode;
+        }
+
+        private static string DownloadTarForWindows()
+        {
+            var tempDir = Path.Combine(Core.BaseDirectory, "temp");
+            AssemblyHandler.EnsureDirectory("temp");
+
+            var libarchiveFile = Path.Combine(tempDir, "libarchive.zip");
+            using WebClient wc = new WebClient();
+            wc.DownloadFile (new Uri(_libArchiveWindowsUrl), libarchiveFile);
+
+            ZipFile.ExtractToDirectory(libarchiveFile, tempDir);
+            var libArchivePath = Path.Combine(tempDir, "libarchive");
+            Directory.Move(Path.Combine(libArchivePath, "bin"), "bsdtar");
+            Directory.Delete(libArchivePath, true);
+            File.Delete(libarchiveFile);
+
+            return Path.Combine(Core.BaseDirectory, "bsdtar/bsdtar.exe");
+        }
+
+        public static bool ExtractToDirectory(
+            string fileNamePath,
+            string outputDirectory,
+            string compressCommand = null,
+            string compressionProgramPath = null
+        )
+        {
+            _pathToTar ??= GetPathToTar();
+
+            var tarFlags = compressCommand == null ? "-axf" : "-xf";
+            var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : "";
+            var arguments = $"{useExternalCompression}{tarFlags} \"{fileNamePath}\" -C \"{outputDirectory}\"";
+
+            return RunTar(arguments, compressionProgramPath) == 0;
+        }
+
+        public static bool CreateFromPaths(
+            List<string> paths,
+            string destinationArchiveFileName,
+            string compressCommand = null,
+            string compressionProgramPath = null
+        )
+        {
+            _pathToTar ??= GetPathToTar();
+
+            AssemblyHandler.EnsureDirectory(new FileInfo(destinationArchiveFileName));
+            var di = new DirectoryInfo(paths[0]);
+            var directory = di.Parent!.FullName;
+
+            using var builder = new ValueStringBuilder();
+            for (var i = 0; i < paths.Count; i++)
+            {
+                var path = paths[i];
+                builder.Append($"{(i > 0 ? " " : "")}\"{Path.GetRelativePath(directory, path)}\"");
+            }
+            var pathsToCompress = builder.ToString();
+
+            var tarFlags = compressCommand == null ? "-acf" : "-cf";
+            var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : "";
+            var arguments = $"{useExternalCompression}{tarFlags} \"{destinationArchiveFileName}\" -C \"{directory}\" {pathsToCompress}";
+
+            return RunTar(arguments, compressionProgramPath) == 0;
+        }
+    }
+}

Projects/UOContent/Compression/ZstdArchive.cs

@@ -0,0 +1,95 @@
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Reflection;
+
+namespace Server.Compression
+{
+    public static class ZstdArchive
+    {
+        private static readonly string _pathToZstd = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory?.FullName;
+
+        public static bool ExtractToDirectory(string fileNamePath, string outputDirectory)
+        {
+            // bsdtar has a bug and hangs, so we are doing it in two steps.
+            if (Core.IsWindows)
+            {
+                var tempTarArchive = Path.Combine(Core.BaseDirectory, "temp/temp-file.tar");
+
+                try
+                {
+                    var process = new Process
+                    {
+                        StartInfo = new ProcessStartInfo
+                        {
+                            FileName = _pathToZstd,
+                            Arguments = $"--no-progress -d \"{fileNamePath}\" -o \"${tempTarArchive}\""
+                        }
+                    };
+
+                    process.Start();
+                    process.WaitForExit();
+
+                    return process.ExitCode == 0 && TarArchive.ExtractToDirectory(tempTarArchive, outputDirectory);
+                }
+                catch
+                {
+                    return false;
+                }
+                finally
+                {
+                    File.Delete(tempTarArchive);
+                }
+            }
+
+            return TarArchive.ExtractToDirectory(fileNamePath, outputDirectory, "zstd -d", _pathToZstd);
+        }
+
+        public static bool CreateFromPaths(
+            List<string> paths,
+            string destinationArchiveFileName,
+            int compressionLevel = 10
+        )
+        {
+            Debug.Assert(compressionLevel is >= 1 and <= 22, $"{nameof(compressionLevel)} must be between 1 and 22");
+
+            // bsdtar has a bug and hangs, so we are doing it in two steps.
+            if (Core.IsWindows)
+            {
+                var tempTarArchive = Path.Combine(Core.BaseDirectory, "temp/temp-file.tar");
+
+                try
+                {
+                    if (!TarArchive.CreateFromPaths(paths, tempTarArchive))
+                    {
+                        return false;
+                    }
+
+                    var process = new Process
+                    {
+                        StartInfo = new ProcessStartInfo
+                        {
+                            FileName = $"{_pathToZstd}\\zstd.exe",
+                            Arguments = $"--no-progress -10 \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\""
+                        }
+                    };
+
+                    process.Start();
+                    process.WaitForExit();
+
+                    return process.ExitCode == 0;
+                }
+                catch
+                {
+                    return false;
+                }
+                finally
+                {
+                    File.Delete(tempTarArchive);
+                }
+            }
+
+            return TarArchive.CreateFromPaths(paths, destinationArchiveFileName, $"zstd -10", _pathToZstd);
+        }
+    }
+}

Projects/UOContent/Misc/AutoRestart.cs

@@ -1,4 +1,5 @@
 using System;
+using Server.Saves;
 
 namespace Server.Misc
 {
@@ -12,15 +13,15 @@ namespace Server.Misc
             get => _enabled;
             set
             {
+                _enabled = value;
+
                 if (value)
                 {
-                    _enabled = true;
                     _autoRestart ??= new AutoRestart();
                     _autoRestart.Start();
                 }
                 else
                 {
-                    _enabled = false;
                     _autoRestart?.Stop();
                     _autoRestart = null;
                 }
@@ -29,21 +30,20 @@ namespace Server.Misc
 
         private static readonly TimeSpan RestartTime = TimeSpan.FromHours(2.0); // time of day at which to restart
 
-        private static readonly TimeSpan
+        // how long the server should remain active before restart (period of 'server wars')
-            RestartDelay =
+        private static readonly TimeSpan RestartDelay = TimeSpan.Zero;
-                TimeSpan.Zero; // how long the server should remain active before restart (period of 'server wars')
 
-        private static readonly TimeSpan
+        // at what interval should the shutdown message be displayed?
-            WarningDelay = TimeSpan.FromMinutes(1.0); // at what interval should the shutdown message be displayed?
+        private static readonly TimeSpan WarningDelay = TimeSpan.FromMinutes(1.0);
 
         private static DateTime m_RestartTime;
 
         public AutoRestart() : base(TimeSpan.FromSeconds(1.0), TimeSpan.FromSeconds(1.0))
         {
 
-            m_RestartTime = DateTime.UtcNow.Date + RestartTime;
+            m_RestartTime = Core.Now.Date + RestartTime;
 
-            if (m_RestartTime < DateTime.UtcNow)
+            if (m_RestartTime < Core.Now)
             {
                 m_RestartTime += TimeSpan.FromDays(1.0);
             }
@@ -71,7 +71,7 @@ namespace Server.Misc
             {
                 e.Mobile.SendMessage("You have initiated server shutdown.");
                 Enabled = true;
-                m_RestartTime = DateTime.UtcNow;
+                m_RestartTime = Core.Now;
             }
         }
 
@@ -92,7 +92,7 @@ namespace Server.Misc
                 return;
             }
 
-            if (DateTime.UtcNow < m_RestartTime)
+            if (Core.Now < m_RestartTime)
             {
                 return;
             }

Projects/UOContent/Misc/AutoSave.cs

@@ -1,114 +0,0 @@
-using System;
-using System.IO;
-using Server.Logging;
-
-namespace Server.Misc
-{
-    public class AutoSave : Timer
-    {
-        private static readonly ILogger logger = LogFactory.GetLogger(typeof(Persistence));
-
-        public static TimeSpan Delay { get; private set; }
-        public static TimeSpan Warning { get; private set; }
-        public static string BackupPath { get; private set; }
-
-        public AutoSave() : base(Delay - Warning, Delay)
-        {
-        }
-
-        public static bool SavesEnabled { get; set; } = true;
-
-        public static void Configure()
-        {
-            BackupPath = ServerConfiguration.GetOrUpdateSetting("autosave.backupPath", "Backups/Automatic");
-            Delay = ServerConfiguration.GetOrUpdateSetting("autosave.saveDelay", TimeSpan.FromMinutes(5.0));
-            Warning = ServerConfiguration.GetOrUpdateSetting("autosave.warningDelay", TimeSpan.Zero);
-        }
-
-        public static void Initialize()
-        {
-            new AutoSave().Start();
-            CommandSystem.Register("SetSaves", AccessLevel.Administrator, SetSaves_OnCommand);
-
-            EventSink.WorldSavePostSnapshot += Backup;
-        }
-
-        [Usage("SetSaves <true | false>"), Description("Enables or disables automatic shard saving.")]
-        public static void SetSaves_OnCommand(CommandEventArgs e)
-        {
-            if (e.Length == 1)
-            {
-                SavesEnabled = e.GetBoolean(0);
-                e.Mobile.SendMessage("Saves have been {0}.", SavesEnabled ? "enabled" : "disabled");
-            }
-            else
-            {
-                e.Mobile.SendMessage("Format: SetSaves <true | false>");
-            }
-        }
-
-        protected override void OnTick()
-        {
-            if (!SavesEnabled || AutoRestart.Restarting)
-            {
-                return;
-            }
-
-            if (Warning == TimeSpan.Zero)
-            {
-                Save();
-            }
-            else
-            {
-                var s = (int)Warning.TotalSeconds;
-                var m = s / 60;
-                s %= 60;
-
-                if (m > 0 && s > 0)
-                {
-                    World.Broadcast(
-                        0x35,
-                        true,
-                        "The world will save in {0} minute{1} and {2} second{3}.",
-                        m,
-                        m != 1 ? "s" : "",
-                        s,
-                        s != 1 ? "s" : ""
-                    );
-                }
-                else if (m > 0)
-                {
-                    World.Broadcast(0x35, true, "The world will save in {0} minute{1}.", m, m != 1 ? "s" : "");
-                }
-                else
-                {
-                    World.Broadcast(0x35, true, "The world will save in {0} second{1}.", s, s != 1 ? "s" : "");
-                }
-
-                StartTimer(Warning, Save);
-            }
-        }
-
-        public static void Save()
-        {
-            if (!AutoRestart.Restarting && World.WorldState == WorldState.Running)
-            {
-                World.Save();
-            }
-        }
-
-        private static void Backup(WorldSavePostSnapshotEventArgs args)
-        {
-            if (!Directory.Exists(args.OldSavePath))
-            {
-                return;
-            }
-
-            var backupPath = Path.Combine(BackupPath, Utility.GetTimeStamp());
-            AssemblyHandler.EnsureDirectory(BackupPath);
-            Directory.Move(args.OldSavePath, backupPath);
-
-            logger.Information($"Created backup at {backupPath}");
-        }
-    }
-}

Projects/UOContent/UOContent.csproj

@@ -30,6 +30,9 @@
         <Delete Files="..\..\Distribution\Assemblies\libz.dylib" ContinueOnError="true" />
         <Delete Files="..\..\Distribution\Assemblies\libz.so" ContinueOnError="true" />
         <Delete Files="..\..\Distribution\Assemblies\ZLib.Bindings.dll" ContinueOnError="true" />
+        <Delete Files="..\..\Distribution\Assemblies\zstd" ContinueOnError="true" />
+        <Delete Files="..\..\Distribution\Assemblies\zstd.exe" ContinueOnError="true" />
+        <Delete Files="..\..\Distribution\Assemblies\Zstd.Binaries.dll" ContinueOnError="true" />
     </Target>
     <ItemGroup>
         <ProjectReference Include="..\Server\Server.csproj" Private="false" PrivateAssets="All" IncludeAssets="None">
@@ -40,6 +43,7 @@
         <PackageReference Include="Microsoft.Toolkit.HighPerformance" Version="7.0.2" />
         <PackageReference Include="Zlib.Bindings" Version="1.5.0" />
         <PackageReference Include="Argon2.Bindings" Version="1.9.1" />
+        <PackageReference Include="Zstd.Binaries" Version="1.0.0" />
     </ItemGroup>
     <ItemGroup>
         <ProjectReference Include="..\SerializationGenerator\SerializationGenerator.csproj">

Projects/UOContent/World Saves/AutoArchive.cs

@@ -0,0 +1,382 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using Server.Compression;
+using Server.Logging;
+
+namespace Server.Saves
+{
+    public enum ArchivePeriod
+    {
+        Hourly,
+        Daily,
+        Monthly
+    }
+
+    public enum CompressionFormat
+    {
+        None,
+        Zip,
+        GZip,
+        Zstd,
+    }
+
+    public static class AutoArchive
+    {
+        private static readonly ILogger logger = LogFactory.GetLogger(typeof(AutoArchive));
+
+        private static int _isArchiving;
+        private static DateTime _nextHourlyArchive;
+        private static DateTime _nextDailyArchive;
+        private static DateTime _nextMonthlyArchive;
+        private static CompressionFormat _compressionFormat;
+        private static bool _enablePruning;
+
+        public static Action Archive { get; set; }
+        public static Action Prune { get; set; }
+        public static string ArchivePath { get; private set; }
+        public static string BackupPath { get; private set; }
+
+        public static void Configure()
+        {
+            BackupPath = ServerConfiguration.GetOrUpdateSetting("autoArchive.backupPath", "Backups/Automatic");
+            ArchivePath = ServerConfiguration.GetOrUpdateSetting("autoArchive.archivePath", "Archives");
+            _compressionFormat = ServerConfiguration.GetOrUpdateSetting("autoArchive.compressionFormat", CompressionFormat.Zstd);
+
+            var useLocalArchives = ServerConfiguration.GetOrUpdateSetting("autoArchive.archiveLocally", true);
+            _enablePruning = ServerConfiguration.GetOrUpdateSetting("autoArchive.enableArchivePruning", true);
+
+            if (useLocalArchives)
+            {
+                Archive = AutoArchiveLocally;
+            }
+
+            if (_enablePruning)
+            {
+                Prune = PruneBackups;
+            }
+
+            // Restores an archive file placed in the Saves folder. Supports all compression formats.
+            RestoreFromArchive();
+
+            DateTimeOffset now = Core.Now;
+            var date = now.Date;
+            _nextHourlyArchive = date.AddHours(now.Hour);
+            _nextDailyArchive = date;
+            _nextMonthlyArchive = date.AddDays(1 - now.Day);
+
+            // Do a local archive rollup & prune
+            AutoArchiveLocally();
+        }
+
+        public static void Initialize()
+        {
+            EventSink.WorldSavePostSnapshot += Backup;
+        }
+
+        private static void Backup(WorldSavePostSnapshotEventArgs args)
+        {
+            if (!Directory.Exists(args.OldSavePath))
+            {
+                return;
+            }
+
+            var backupPath = Path.Combine(BackupPath, Utility.GetTimeStamp());
+            AssemblyHandler.EnsureDirectory(BackupPath);
+            Directory.Move(args.OldSavePath, backupPath);
+
+            logger.Information($"Created backup at {backupPath}");
+
+            Archive?.Invoke();
+        }
+
+        private static void RestoreFromArchive()
+        {
+            var savePath = Path.Combine(Core.BaseDirectory, ServerConfiguration.GetSetting("world.savePath", "Saves"));
+
+            var files = Directory.Exists(savePath) ? Directory.GetFiles(savePath, "????-??-??-??-??-??.*") : null;
+
+            if (files == null || files.Length == 0)
+            {
+                return;
+            }
+
+            var latestFiles = GetInRange(files, DateTime.MinValue, DateTime.MaxValue, true);
+
+            foreach (var file in latestFiles)
+            {
+                if (RestoreFromFile(file, savePath))
+                {
+                    break;
+                }
+            }
+        }
+
+        private static bool RestoreFromFile(string file, string savePath)
+        {
+            var fi = new FileInfo(file);
+            var fileName = fi.Name;
+
+            if (!TryGetDate(fileName[..fileName.IndexOfOrdinal(".")], out _))
+            {
+                return false;
+            }
+
+            logger.Information($"Restoring latest world save from archive {fileName}");
+
+            var tempFolder = Path.Combine(Core.BaseDirectory, "temp");
+            AssemblyHandler.EnsureDirectory(tempFolder);
+            bool successful;
+
+            if (fileName.EndsWithOrdinal(".tar.zst"))
+            {
+                successful = ZstdArchive.ExtractToDirectory(fi.FullName, tempFolder);
+            }
+            else
+            {
+                TarArchive.ExtractToDirectory(fi.FullName, tempFolder);
+                successful = true;
+            }
+
+            if (!successful)
+            {
+                logger.Information($"Failed to extract {fi.Name}");
+                return false;
+            }
+
+            var allFolders = Directory.EnumerateDirectories(tempFolder, "????-??-??-??-??-??");
+            var worldSaveFolders = GetInRange(allFolders, DateTime.MinValue, DateTime.MaxValue);
+            var first = true;
+            foreach (var folder in worldSaveFolders)
+            {
+                if (first)
+                {
+                    first = false;
+                    Directory.Delete(savePath, true);
+                    var dirInfo = new DirectoryInfo(folder);
+                    logger.Information($"Restoring backup {dirInfo.Name}");
+                    Directory.Move(folder, savePath);
+                }
+                else
+                {
+                    Directory.Delete(folder, true);
+                }
+            }
+
+            return true;
+        }
+
+        public static void PruneBackups()
+        {
+            // Maintain a total of 66 of the most recent archives
+            PruneLocalArchives(ArchivePeriod.Hourly, 24);
+            PruneLocalArchives(ArchivePeriod.Daily, 30);
+            PruneLocalArchives(ArchivePeriod.Monthly, 12);
+
+            var allFolders = Directory.EnumerateDirectories(BackupPath, "????-??-??-??-??-??", SearchOption.AllDirectories);
+            var latestBackupFolders = GetInRange(allFolders, DateTime.MinValue, Core.Now.AddMonths(-1));
+
+            foreach (var folder in latestBackupFolders)
+            {
+                logger.Information($"Pruning backup {folder}");
+                Directory.Delete(folder, true);
+            }
+        }
+
+        private static void PruneLocalArchives(ArchivePeriod period, int minRetained)
+        {
+            var periodStr = period.ToString();
+            var path = Path.Combine(ArchivePath, periodStr);
+            var allFiles = Directory.EnumerateFiles(path, "????-??-??-??-??-??.*");
+            var archives = GetInRange(allFiles, DateTime.MinValue, DateTime.MaxValue, true);
+
+            var periodLowerStr = periodStr.ToLowerInvariant();
+            foreach (var archive in archives)
+            {
+                if (minRetained > 0)
+                {
+                    minRetained--;
+                    continue;
+                }
+
+                var fi = new FileInfo(archive);
+                logger.Information($"Pruning {periodLowerStr} archive {fi.Name}");
+                File.Delete(archive);
+            }
+        }
+
+        public static void AutoArchiveLocally()
+        {
+            var date = Core.Now;
+
+            if (date < _nextHourlyArchive && date < _nextDailyArchive && date < _nextMonthlyArchive)
+            {
+                return;
+            }
+
+            if (Interlocked.CompareExchange(ref _isArchiving, 0, 1) == 1)
+            {
+                return;
+            }
+
+            ThreadPool.UnsafeQueueUserWorkItem(
+                _ =>
+                {
+                    if (date >= _nextHourlyArchive)
+                    {
+                        var lastHour = date.Date.AddHours(date.Hour);
+                        Rollup(ArchivePeriod.Hourly, lastHour.AddHours(-1), lastHour.ToTimeStamp());
+                        _nextHourlyArchive = _nextHourlyArchive.AddHours(2);
+                    }
+
+                    if (date >= _nextDailyArchive)
+                    {
+                        var yesterday = date.Date.AddDays(-1);
+                        Rollup(ArchivePeriod.Daily, yesterday, yesterday.ToTimeStamp());
+                        _nextDailyArchive = _nextDailyArchive.AddDays(2);
+                    }
+
+                    if (date >= _nextMonthlyArchive)
+                    {
+                        var lastMonth = new DateTime(date.Year, date.Month, 1).AddMonths(-1);
+                        Rollup(ArchivePeriod.Monthly, lastMonth, lastMonth.ToTimeStamp());
+                        _nextMonthlyArchive = _nextMonthlyArchive.AddMonths(2);
+                    }
+
+                    Prune?.Invoke();
+
+                    _isArchiving = 0;
+                },
+                null
+            );
+        }
+
+        private static void Rollup(ArchivePeriod archivePeriod, DateTime rangeStart, string archiveNameNoExtension)
+        {
+            var archivePeriodStr = archivePeriod.ToString();
+            var archivePeriodStrLower = archivePeriodStr.ToLowerInvariant();
+
+            var stopWatch = new Stopwatch();
+            stopWatch.Start();
+            var allFolders = Directory.EnumerateDirectories(BackupPath, "????-??-??-??-??-??");
+
+            var rangeEnd = archivePeriod switch
+            {
+                ArchivePeriod.Monthly => rangeStart.AddMonths(1),
+                ArchivePeriod.Daily   => rangeStart.AddDays(1),
+                _  => rangeStart.AddHours(1),
+            };
+
+            var latestFolders = GetInRange(allFolders, rangeStart, rangeEnd).ToList();
+
+            // We don't want to re-archive a single save. This usually means we rebooted and found the one we purposefully left behind
+            if (latestFolders.Count <= 1)
+            {
+                return;
+            }
+
+            var extension = _compressionFormat.GetFileExtension();
+            var archiveFilePath = Path.Combine(ArchivePath, archivePeriodStr, $"{archiveNameNoExtension}{extension}");
+
+            logger.Information($"Creating {archivePeriodStrLower} archive");
+            var archiveCreated = _compressionFormat == CompressionFormat.Zstd
+                ? ZstdArchive.CreateFromPaths(latestFolders, archiveFilePath)
+                : TarArchive.CreateFromPaths(latestFolders, archiveFilePath);
+
+            if (archiveCreated)
+            {
+                stopWatch.Stop();
+                var elapsed = stopWatch.Elapsed.TotalSeconds;
+                logger.Information($"Created {archivePeriodStrLower} archive at {archiveFilePath} ({elapsed:F2} seconds)");
+
+                // Keep the latest one, but delete the rest.
+                for (var i = 1; i < latestFolders.Count; i++)
+                {
+                    Directory.Delete(latestFolders[i], true);
+                }
+            }
+            else
+            {
+                logger.Warning($"Failed to create {archivePeriodStrLower} archive");
+            }
+        }
+
+        private static IEnumerable<string> GetInRange(
+            IEnumerable<string> allItems,
+            DateTime rangeStart,
+            DateTime rangeEnd,
+            bool files = false
+        )
+        {
+            var items = new SortedDictionary<DateTime, string>(new DescendingComparer<DateTime>());
+            foreach (var item in allItems)
+            {
+                string name;
+                if (files)
+                {
+                    var fileName = new FileInfo(item).Name;
+                    name = fileName[..fileName.IndexOfOrdinal(".")];
+                }
+                else
+                {
+                    name = new DirectoryInfo(item).Name;
+                }
+
+                if (IsInRange(name, rangeStart, rangeEnd, out var date))
+                {
+                    items.Add(date, item);
+                }
+            }
+
+            return items.Values;
+        }
+
+        private static bool IsInRange(string name, DateTime start, DateTime end, out DateTime date) =>
+            TryGetDate(name, out date) && start <= date && end >= date;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static bool TryGetDate(string value, out DateTime date)
+        {
+            // Expecting YYYY-MM-DD-HH-mm-ss
+            var split = value.Split("-");
+            if (split.Length != 6)
+            {
+                date = DateTime.MinValue;
+                return false;
+            }
+
+            try
+            {
+                date = new DateTime(
+                    int.Parse(split[0]),
+                    int.Parse(split[1]),
+                    int.Parse(split[2]),
+                    int.Parse(split[3]),
+                    int.Parse(split[4]),
+                    int.Parse(split[5])
+                );
+            }
+            catch
+            {
+                date = DateTime.MinValue;
+                return false;
+            }
+
+            return true;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static string GetFileExtension(this CompressionFormat compressionFormat) =>
+            compressionFormat switch
+            {
+                CompressionFormat.Zip  => ".zip",
+                CompressionFormat.GZip => ".tar.gz",
+                CompressionFormat.Zstd => ".tar.zst",
+                _                      => ".tar"
+            };
+    }
+}

Projects/UOContent/World Saves/AutoSave.cs

@@ -0,0 +1,132 @@
+using System;
+using Server.Misc;
+
+namespace Server.Saves
+{
+    public class AutoSave : Timer
+    {
+        private static Timer _autoSave;
+        private static bool _enabled;
+        private static DateTime _nextSave;
+
+        public static TimeSpan Delay { get; private set; }
+        public static TimeSpan Warning { get; private set; }
+
+        public static bool SavesEnabled
+        {
+            get => _enabled;
+            set
+            {
+                _enabled = value;
+
+                if (value)
+                {
+                    SetNextSave(Core.Now.ToSystemLocalTime());
+                    _autoSave ??= new AutoSave();
+                    _autoSave.Start();
+                }
+                else
+                {
+                    _autoSave?.Stop();
+                    _autoSave = null;
+                }
+            }
+        }
+
+        public static void Configure()
+        {
+            Delay = ServerConfiguration.GetOrUpdateSetting("autosave.saveDelay", TimeSpan.FromMinutes(5.0));
+            Warning = ServerConfiguration.GetOrUpdateSetting("autosave.warningDelay", TimeSpan.Zero);
+        }
+
+        public static void Initialize()
+        {
+            SavesEnabled = true;
+        }
+
+        public static void ResetAutoSave(TimeSpan saveDelay, TimeSpan warningDelay)
+        {
+            if (saveDelay != Delay || warningDelay != Warning)
+            {
+                Delay = saveDelay;
+                Warning = warningDelay;
+
+                ServerConfiguration.SetSetting("autosave.saveDelay", Delay);
+                ServerConfiguration.SetSetting("autosave.warningDelay", Warning);
+            }
+
+            SavesEnabled = true;
+        }
+
+        private static void SetNextSave(DateTime now)
+        {
+            var timeOfDay = now.TimeOfDay;
+            var delay = Delay.Ticks;
+
+            _nextSave = (now + TimeSpan.FromTicks(delay - timeOfDay.Ticks % delay) - Warning).ToUniversalTime();
+        }
+
+        public AutoSave() : base(TimeSpan.FromSeconds(1.0), TimeSpan.FromSeconds(1.0))
+        {
+        }
+
+        protected override void OnTick()
+        {
+            if (_nextSave > Core.Now)
+            {
+                return;
+            }
+
+            if (SavesEnabled && !AutoRestart.Restarting && !Core.Closing && World.Running)
+            {
+                if (Warning > TimeSpan.Zero)
+                {
+                    BroadcastWarning();
+                    StartTimer(Warning, Save);
+                }
+                else
+                {
+                    Save();
+                }
+            }
+
+            _nextSave += Delay;
+        }
+
+        private static void BroadcastWarning()
+        {
+            var s = (int)Warning.TotalSeconds;
+            var m = s / 60;
+            s %= 60;
+
+            if (m > 0 && s > 0)
+            {
+                World.Broadcast(
+                    0x35,
+                    true,
+                    "The world will save in {0} minute{1} and {2} second{3}.",
+                    m,
+                    m != 1 ? "s" : "",
+                    s,
+                    s != 1 ? "s" : ""
+                );
+            }
+            else if (m > 0)
+            {
+                World.Broadcast(0x35, true, "The world will save in {0} minute{1}.", m, m != 1 ? "s" : "");
+            }
+            else
+            {
+                World.Broadcast(0x35, true, "The world will save in {0} second{1}.", s, s != 1 ? "s" : "");
+            }
+        }
+
+        public static void Save()
+        {
+            if (!AutoRestart.Restarting && World.Running)
+            {
+                World.Save();
+            }
+        }
+    }
+}

Projects/UOContent/World Saves/SaveCommands.cs

@@ -0,0 +1,60 @@
+using Server.Saves;
+
+namespace Server.Commands
+{
+    public static class SaveCommands
+    {
+        public static void Initialize()
+        {
+            CommandSystem.Register("Save", AccessLevel.Administrator, Save_OnCommand);
+            CommandSystem.Register("SetSaves", AccessLevel.Administrator, SetAutoSaves_OnCommand);
+            CommandSystem.Register("AutoSave", AccessLevel.Administrator, SetAutoSaves_OnCommand);
+            CommandSystem.Register("SaveFrequency", AccessLevel.Administrator, SetSaveFrequency_OnCommand);
+            CommandSystem.Register("PruneArchives", AccessLevel.Administrator, PruneArchives_OnCommand);
+        }
+
+        [Usage("PruneArchives"), Description("Prunes archives folder.")]
+        private static void PruneArchives_OnCommand(CommandEventArgs e)
+        {
+            AutoArchive.PruneBackups();
+        }
+
+        [Usage("Save"), Description("Saves the world.")]
+        private static void Save_OnCommand(CommandEventArgs e)
+        {
+            AutoSave.Save();
+        }
+
+        [Usage("AutoSave <on | off>"), Description("Enables or disables automatic shard saving.")]
+        public static void SetAutoSaves_OnCommand(CommandEventArgs e)
+        {
+            if (e.Length == 1)
+            {
+
+                var enabled = AutoSave.SavesEnabled = e.GetBoolean(0);
+                e.Mobile.SendMessage("Saves have been {0}.", enabled ? "enabled" : "disabled");
+            }
+            else
+            {
+                e.Mobile.SendMessage("Format: AutoSave <on | off>");
+            }
+        }
+
+        [Usage("SaveFrequency <duration> [warning duration]"), Description("Sets the save frequency starting at midnight local to the server.")]
+        public static void SetSaveFrequency_OnCommand(CommandEventArgs e)
+        {
+            if (e.Length < 1)
+            {
+                e.Mobile.SendMessage("Format: SaveFrequency <duration> [warning duration]");
+                return;
+            }
+
+            var saveDelay = e.GetTimeSpan(0);
+            var warningDelay = e.Length >= 2 ? e.GetTimeSpan(1) : AutoSave.Warning;
+
+            AutoSave.ResetAutoSave(saveDelay, warningDelay);
+
+            e.Mobile.SendMessage("Save frequency has been updated.");
+        }
+    }
+}