warrentyexpired/ModernUO
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix: Cleans up file system paths and archiving (#815)

Browse source 
* Makes EnsureDirectory properly work for relative and absolute paths
* Adds a `PathUtility.GetFullPath` which returns full paths for relative paths to `Core.BaseDirectory`. If the path is absolute, it will return as-is.
* Moves EnsureDirectory to `PathUtility`. So `ScriptsHandler.EnsureDirectory` and `AssemblyHandler.EnsureDirectory` are now `PathUtility.EnsureDirectory`
* Fixes crash guard so that it copies accounts properly.
* Changes world save and auto archive to use a random folder name inside of the temp folder.
This commit is contained in:
Kamron Batman 2021-10-05 08:54:44 -07:00 • committed by GitHub
parent da31153b20
commit a55e271a69
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 447 additions and 341 deletions
Download patch file Download diff file Expand all files Collapse all files

22
Projects/Server/AssemblyHandler.cs
View file

@ -173,28 +173,6 @@ namespace Server
return null;
}
public static string EnsureDirectory(string dir)
{
var path = Path.Combine(Core.BaseDirectory, dir);
Directory.CreateDirectory(path);
return path;
}
public static void EnsureDirectory(FileInfo fi)
{
var dir = fi.DirectoryName;
if (dir != null)
{
Directory.CreateDirectory(dir);
}
}
public static void EnsureDirectory(DirectoryInfo di)
{
Directory.CreateDirectory(di.FullName);
}
}
public class TypeCache

4
Projects/Server/Serialization/GenericPersistence.cs
View file

@ -41,7 +41,7 @@ namespace Server
{
var path = Path.Combine(savePath, name);
AssemblyHandler.EnsureDirectory(path);
PathUtility.EnsureDirectory(path);
string binPath = Path.Combine(path, $"{name}.bin");
using var bin = new BinaryFileWriter(binPath, true);
@ -54,7 +54,7 @@ namespace Server
{
var path = Path.Combine(savePath, name);
AssemblyHandler.EnsureDirectory(path);
PathUtility.EnsureDirectory(path);
string binPath = Path.Combine(path, $"{name}.bin");

22
Projects/Server/Text/StringHelpers.cs
View file

@ -169,6 +169,28 @@ namespace Server
return str;
}
public static string TrimMultiline(this string str, string lineSeparator = "\n")
{
var parts = str.Split(lineSeparator);
for (var i = 0; i < parts.Length; i++)
{
parts[i] = parts[i].Trim();
}
return string.Join(lineSeparator, parts);
}
public static string IndentMultiline(this string str, string indent = "\t", string lineSeparator = "\n")
{
var parts = str.Split(lineSeparator);
for (var i = 0; i < parts.Length; i++)
{
parts[i] = $"{indent}{parts[i]}";
}
return string.Join(lineSeparator, parts);
}
public static List<string> Wrap(this string value, int perLine, int maxLines)
{
if ((value = value?.Trim() ?? "").Length <= 0)

2
Projects/Server/Timer/Timer.TimerWheel.cs
View file

@ -242,7 +242,7 @@ namespace Server
}
#if DEBUG_TIMERS
tw.WriteLine("\nStack Traces:");
tw.WriteLine($"{Environment.NewLine}Stack Traces:");
foreach (var kvp in DelayCallTimer._stackTraces)
{
tw.WriteLine(kvp.Value);

67
Projects/Server/Utilities/PathUtility.cs Normal file
View file

@ -0,0 +1,67 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2021 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: PathUtility.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
using System;
using System.IO;
using Server.Text;
namespace Server
{
public static class PathUtility
{
public static string EnsureDirectory(string dir)
{
var path = GetFullPath(dir, Core.BaseDirectory);
Directory.CreateDirectory(path);
return path;
}
public static void EnsureDirectory(this FileInfo fi)
{
var dir = GetFullPath(fi.DirectoryName, Core.BaseDirectory);
if (dir != null)
{
Directory.CreateDirectory(dir);
}
}
public static void EnsureDirectory(this DirectoryInfo di)
{
var file = GetFullPath(di.FullName, Core.BaseDirectory);
Directory.CreateDirectory(file);
}
public static string GetFullPath(string relativeOrAbsolutePath) =>
GetFullPath(relativeOrAbsolutePath, Core.BaseDirectory);
public static string GetFullPath(string relativeOrAbsolutePath, string basePath) =>
relativeOrAbsolutePath switch
{
null => null,
"" => basePath,
_ => Path.IsPathRooted(relativeOrAbsolutePath)
? relativeOrAbsolutePath
: Path.GetFullPath(relativeOrAbsolutePath, basePath)
};
public static string EnsureRandomPath(string basePath)
{
Span<byte> bytes = stackalloc byte[8];
Utility.RandomBytes(bytes);
return EnsureDirectory(Path.Combine(basePath, bytes.ToHexString()));
}
}
}

22
Projects/Server/Utilities/Utility.cs
View file

@ -1331,28 +1331,6 @@ namespace Server
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T Max<T>(T val, T max) where T : IComparable<T> => val.CompareTo(max) > 0 ? val : max;
public static string TrimMultiline(this string str, string lineSeparator = "\n")
{
var parts = str.Split(lineSeparator);
for (var i = 0; i < parts.Length; i++)
{
parts[i] = parts[i].Trim();
}
return string.Join(lineSeparator, parts);
}
public static string IndentMultiline(this string str, string indent = "\t", string lineSeparator = "\n")
{
var parts = str.Split(lineSeparator);
for (var i = 0; i < parts.Length; i++)
{
parts[i] = $"{indent}{parts[i]}";
}
return string.Join(lineSeparator, parts);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Tidy<T>(this List<T> list) where T : ISerializable
{

2
Projects/Server/World/EntityPersistence.cs
View file

@ -40,7 +40,7 @@ namespace Server
var path = Path.Combine(savePath, typeName);
AssemblyHandler.EnsureDirectory(path);
PathUtility.EnsureDirectory(path);
string idxPath = Path.Combine(path, $"{typeName}.idx");
string tdbPath = Path.Combine(path, $"{typeName}.tdb");

21
Projects/Server/World/World.cs
View file

@ -46,7 +46,6 @@ namespace Server
private static readonly ConcurrentQueue<Item> _decayQueue = new();
private static string _tempSavePath; // Path to the temporary folder for the save
private static string _savePath; // Path to "Saves" folder
private static bool _enableSaveStats;
public const bool DirtyTrackingEnabled = false;
@ -130,6 +129,8 @@ namespace Server
internal static List<Type> MobileTypes { get; } = new();
internal static List<Type> GuildTypes { get; } = new();
public static string SavePath { get; private set; }
public static WorldState WorldState { get; private set; }
public static bool Saving => WorldState == WorldState.Saving;
public static bool Running => WorldState is not WorldState.Loading and not WorldState.Initial;
@ -141,10 +142,12 @@ namespace Server
public static void Configure()
{
var tempSavePath = ServerConfiguration.GetOrUpdateSetting("world.tempSavePath", "temp");
_tempSavePath = Path.Combine(Core.BaseDirectory, tempSavePath);
var tempSavePath = ServerConfiguration.GetSetting("world.tempSavePath", "temp");
_tempSavePath = PathUtility.GetFullPath(tempSavePath);
var savePath = ServerConfiguration.GetOrUpdateSetting("world.savePath", "Saves");
_savePath = Path.Combine(Core.BaseDirectory, savePath);
SavePath = PathUtility.GetFullPath(savePath);
_enableSaveStats = ServerConfiguration.GetOrUpdateSetting("world.enableSaveStats", false);
// Mobiles & Items
@ -256,7 +259,7 @@ namespace Server
logger.Information("Loading world");
var watch = Stopwatch.StartNew();
Persistence.Load(_savePath);
Persistence.Load(SavePath);
EventSink.InvokeWorldLoad();
ProcessSafetyQueues();
@ -344,7 +347,7 @@ namespace Server
var timestamp = Utility.GetTimeStamp();
var saveStatsPath = Path.Combine(Core.BaseDirectory, $"Logs/Saves/Save-Stats-{timestamp}.log");
AssemblyHandler.EnsureDirectory(saveStatsPath);
PathUtility.EnsureDirectory(saveStatsPath);
using var op = new StreamWriter(saveStatsPath, true);
@ -388,7 +391,7 @@ namespace Server
{
Exception exception = null;
var tempPath = Path.Combine(_tempSavePath, Utility.GetTimeStamp());
var tempPath = PathUtility.EnsureRandomPath(_tempSavePath);
try
{
@ -417,8 +420,8 @@ namespace Server
{
try
{
EventSink.InvokeWorldSavePostSnapshot(_savePath, tempPath);
Directory.Move(tempPath, _savePath);
EventSink.InvokeWorldSavePostSnapshot(SavePath, tempPath);
Directory.Move(tempPath, SavePath);
}
catch (Exception ex)
{

3
Projects/UOContent.Tests/Fixtures/ServerFixture.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Reflection;
using Server.Misc;
namespace Server.Tests
@ -8,8 +9,8 @@ namespace Server.Tests
// Global setup
static ServerFixture()
{
Core.Assembly = Assembly.GetExecutingAssembly();
Core.LoopContext = new EventLoopContext();
Core.Expansion = Expansion.EJ;
// Load Configurations

101
Projects/UOContent/Accounting/Account.Migrations.cs Normal file
View file

@ -0,0 +1,101 @@
using System.Collections.Generic;
using System.IO;
using System.Net;
using Server.Accounting.Security;
namespace Server.Accounting
{
public partial class Account
{
private void MigrateFrom(V3Content content)
{
_username = content.Username;
_passwordAlgorithm = content.PasswordAlgorithm;
_password = content.Password;
_accessLevel = content.AccessLevel;
_flags = content.Flags;
Created = content.Created;
_lastLogin = content.LastLogin;
_totalGold = content.TotalGold;
_totalPlat = content.TotalPlat;
_mobiles = content.Mobiles;
_comments = content.Comments;
_tags = content.Tags;
_loginIPs = content.LoginIPs;
_ipRestrictions = content.IpRestrictions;
_totalGameTime = content.TotalGameTime;
_email = content.Email;
}
private void Deserialize(IGenericReader reader, int version)
{
if (version != 2)
{
// Due to a bug where we were not versioning at all, reset so we don't have an issue deserializing
reader.Seek(0, SeekOrigin.Begin);
}
_username = reader.ReadString();
_passwordAlgorithm = version < 2 ? (PasswordProtectionAlgorithm)reader.ReadInt() : reader.ReadEnum<PasswordProtectionAlgorithm>();
_password = reader.ReadString();
_accessLevel = version < 2 ? (AccessLevel)reader.ReadInt() : reader.ReadEnum<AccessLevel>();
_flags = reader.ReadInt();
Created = reader.ReadDateTime();
_lastLogin = reader.ReadDateTime();
_totalGold = reader.ReadInt();
_totalPlat = reader.ReadInt();
var length = reader.ReadInt();
_mobiles = new Mobile[length];
for (int i = 0; i < length; i++)
{
_mobiles[i] = reader.ReadEntity<Mobile>();
}
length = reader.ReadInt();
_comments = length > 0 ? new List<AccountComment>(length) : null;
for (int i = 0; i < length; i++)
{
_comments!.Add(new AccountComment(reader));
}
length = reader.ReadInt();
_tags = length > 0 ? new List<AccountTag>(length) : null;
for (int i = 0; i < length; i++)
{
_tags!.Add(new AccountTag(reader));
}
length = reader.ReadInt();
_loginIPs = new IPAddress[length];
for (int i = 0; i < length; i++)
{
if (version < 2)
{
if (IPAddress.TryParse(reader.ReadString(), out var address))
{
_loginIPs[i] = Utility.Intern(address);
}
}
else
{
_loginIPs[i] = reader.ReadIPAddress();
}
}
length = reader.ReadInt();
_ipRestrictions = new string[length];
for (int i = 0; i < length; i++)
{
_ipRestrictions[i] = reader.ReadString();
}
_totalGameTime = reader.ReadTimeSpan();
_email = reader.ReadString();
Timer.StartTimer(AfterDeserialization);
}
}
}

92
Projects/UOContent/Accounting/Account.cs
View file

@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Xml;
using Server.Accounting.Security;
@ -357,97 +356,6 @@ namespace Server.Accounting
}
}
private void MigrateFrom(V3Content content)
{
_username = content.Username;
_passwordAlgorithm = content.PasswordAlgorithm;
_password = content.Password;
_accessLevel = content.AccessLevel;
_flags = content.Flags;
Created = content.Created;
_lastLogin = content.LastLogin;
_totalGold = content.TotalGold;
_totalPlat = content.TotalPlat;
_mobiles = content.Mobiles;
_comments = content.Comments;
_tags = content.Tags;
_loginIPs = content.LoginIPs;
_ipRestrictions = content.IpRestrictions;
_totalGameTime = content.TotalGameTime;
_email = content.Email;
}
private void Deserialize(IGenericReader reader, int version)
{
if (version != 2)
{
// Due to a bug where we were not versioning at all, reset so we don't have an issue deserializing
reader.Seek(0, SeekOrigin.Begin);
}
_username = reader.ReadString();
_passwordAlgorithm = version < 2 ? (PasswordProtectionAlgorithm)reader.ReadInt() : reader.ReadEnum<PasswordProtectionAlgorithm>();
_password = reader.ReadString();
_accessLevel = version < 2 ? (AccessLevel)reader.ReadInt() : reader.ReadEnum<AccessLevel>();
_flags = reader.ReadInt();
Created = reader.ReadDateTime();
_lastLogin = reader.ReadDateTime();
_totalGold = reader.ReadInt();
_totalPlat = reader.ReadInt();
var length = reader.ReadInt();
_mobiles = new Mobile[length];
for (int i = 0; i < length; i++)
{
_mobiles[i] = reader.ReadEntity<Mobile>();
}
length = reader.ReadInt();
_comments = length > 0 ? new List<AccountComment>(length) : null;
for (int i = 0; i < length; i++)
{
_comments!.Add(new AccountComment(reader));
}
length = reader.ReadInt();
_tags = length > 0 ? new List<AccountTag>(length) : null;
for (int i = 0; i < length; i++)
{
_tags!.Add(new AccountTag(reader));
}
length = reader.ReadInt();
_loginIPs = new IPAddress[length];
for (int i = 0; i < length; i++)
{
if (version < 2)
{
if (IPAddress.TryParse(reader.ReadString(), out var address))
{
_loginIPs[i] = Utility.Intern(address);
}
}
else
{
_loginIPs[i] = reader.ReadIPAddress();
}
}
length = reader.ReadInt();
_ipRestrictions = new string[length];
for (int i = 0; i < length; i++)
{
_ipRestrictions[i] = reader.ReadString();
}
_totalGameTime = reader.ReadTimeSpan();
_email = reader.ReadString();
Timer.StartTimer(AfterDeserialization);
}
/// <summary>
/// Deletes the account, all characters of the account, and all houses of those characters
/// </summary>

18
Projects/UOContent/Compression/TarArchive.cs
View file

@ -50,8 +50,7 @@ namespace Server.Compression
private static string DownloadTarForWindows()
{
var tempDir = Path.Combine(Core.BaseDirectory, "temp");
AssemblyHandler.EnsureDirectory("temp");
var tempDir = PathUtility.EnsureRandomPath(Path.GetTempPath());
var libarchiveFile = Path.Combine(tempDir, "libarchive.zip");
using WebClient wc = new WebClient();
@ -82,29 +81,28 @@ namespace Server.Compression
}
public static bool CreateFromPaths(
List<string> paths,
IEnumerable<string> paths,
string destinationArchiveFileName,
string relativeTo,
string compressCommand = null,
string compressionProgramPath = null
)
{
_pathToTar ??= GetPathToTar();
AssemblyHandler.EnsureDirectory(new FileInfo(destinationArchiveFileName));
var di = new DirectoryInfo(paths[0]);
var directory = di.Parent!.FullName;
new FileInfo(destinationArchiveFileName).EnsureDirectory();
using var builder = new ValueStringBuilder();
for (var i = 0; i < paths.Count; i++)
var i = 0;
foreach (var path in paths)
{
var path = paths[i];
builder.Append($"{(i > 0 ? " " : "")}\"{Path.GetRelativePath(directory, path)}\"");
builder.Append($"{(i++ > 0 ? " " : "")}\"{Path.GetRelativePath(relativeTo, path)}\"");
}
var pathsToCompress = builder.ToString();
var tarFlags = compressCommand == null ? "-acf" : "-cf";
var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : "";
var arguments = $"{useExternalCompression}{tarFlags} \"{destinationArchiveFileName}\" -C \"{directory}\" {pathsToCompress}";
var arguments = $"{useExternalCompression}{tarFlags} \"{destinationArchiveFileName}\" -C \"{relativeTo}\" {pathsToCompress}";
return RunTar(arguments, compressionProgramPath) == 0;
}

14
Projects/UOContent/Compression/ZstdArchive.cs
View file

@ -42,17 +42,15 @@ namespace Server.Compression
}
}
return TarArchive.ExtractToDirectory(fileNamePath, outputDirectory, "zstd -d", _pathToZstd);
return TarArchive.ExtractToDirectory(fileNamePath, outputDirectory, "zstd -q -d", _pathToZstd);
}
public static bool CreateFromPaths(
List<string> paths,
IEnumerable<string> paths,
string destinationArchiveFileName,
int compressionLevel = 10
string relativeTo
)
{
Debug.Assert(compressionLevel is >= 1 and <= 22, $"{nameof(compressionLevel)} must be between 1 and 22");
// bsdtar has a bug and hangs, so we are doing it in two steps.
if (Core.IsWindows)
{
@ -60,7 +58,7 @@ namespace Server.Compression
try
{
if (!TarArchive.CreateFromPaths(paths, tempTarArchive))
if (!TarArchive.CreateFromPaths(paths, relativeTo, tempTarArchive))
{
return false;
}
@ -70,7 +68,7 @@ namespace Server.Compression
StartInfo = new ProcessStartInfo
{
FileName = Path.Combine(_pathToZstd, "zstd.exe"),
Arguments = $"-q -10 \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\""
Arguments = $"-q \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\""
}
};
@ -89,7 +87,7 @@ namespace Server.Compression
}
}
return TarArchive.CreateFromPaths(paths, destinationArchiveFileName, "zstd -10", _pathToZstd);
return TarArchive.CreateFromPaths(paths, destinationArchiveFileName, relativeTo, "zstd -q", _pathToZstd);
}
}
}

2
Projects/UOContent/Engines/Spawners/Commands/ConvertPremiumSpawners.cs
View file

@ -77,7 +77,7 @@ namespace Server.Engines.Spawners
{
var stemDir = stem[..^file.Name.Length];
var fullOutputDir = Path.Combine(outputDir, stemDir);
AssemblyHandler.EnsureDirectory(fullOutputDir);
PathUtility.EnsureDirectory(fullOutputDir);
var outputFileName = $"{file.Name[..^file.Extension.Length]}.json";
ParsePremiumSpawnerFile(file.FullName, Path.Combine(fullOutputDir, outputFileName), options);

2
Projects/UOContent/Engines/Spawners/Commands/ExportSpawnersCommand.cs
View file

@ -53,7 +53,7 @@ namespace Server.Engines.Spawners
if (!Path.IsPathRooted(path))
{
path = Path.Combine(Core.BaseDirectory, path);
AssemblyHandler.EnsureDirectory(directory);
PathUtility.EnsureDirectory(directory);
}
else if (!Directory.Exists(directory))
{

76
Projects/UOContent/Misc/CrashGuard.cs
View file

@ -4,6 +4,7 @@ using System.IO;
using Server.Accounting;
using Server.Logging;
using Server.Network;
using Server.Saves;
namespace Server.Misc
{
@ -76,24 +77,23 @@ namespace Server.Misc
}
}
private static void CopyFile(string rootOrigin, string rootBackup, string path)
private static void DirectoryCopy(string sourceDirName, string destDirName)
{
var originPath = Path.Combine(rootOrigin, path);
if (!File.Exists(originPath))
// Get the subdirectories for the specified directory.
DirectoryInfo dir = new DirectoryInfo(sourceDirName);
DirectoryInfo[] dirs = dir.GetDirectories();
FileInfo[] files = dir.GetFiles();
foreach (FileInfo file in files)
{
return;
string tempPath = Path.Combine(destDirName, file.Name);
file.CopyTo(tempPath, false);
}
var backupPath = Path.Combine(rootBackup, path);
Directory.CreateDirectory(Path.GetDirectoryName(backupPath));
try
foreach (DirectoryInfo subdir in dirs)
{
File.Copy(originPath, backupPath);
}
catch
{
// ignored
string tempPath = Path.Combine(destDirName, subdir.Name);
DirectoryCopy(subdir.FullName, tempPath);
}
}
@ -105,26 +105,12 @@ namespace Server.Misc
{
var timeStamp = Utility.GetTimeStamp();
var root = Core.BaseDirectory;
var rootBackup = Path.Combine(root, $"Backups/Crashed/{timeStamp}/");
var rootOrigin = Path.Combine(root, "Saves/");
// Copy files
CopyFile(rootOrigin, rootBackup, "Accounts/Accounts.xml");
CopyFile(rootOrigin, rootBackup, "Items/Items.bin");
CopyFile(rootOrigin, rootBackup, "Items/Items.idx");
CopyFile(rootOrigin, rootBackup, "Items/Items.tdb");
CopyFile(rootOrigin, rootBackup, "Mobiles/Mobiles.bin");
CopyFile(rootOrigin, rootBackup, "Mobiles/Mobiles.idx");
CopyFile(rootOrigin, rootBackup, "Mobiles/Mobiles.tdb");
CopyFile(rootOrigin, rootBackup, "Guilds/Guilds.bin");
CopyFile(rootOrigin, rootBackup, "Guilds/Guilds.idx");
CopyFile(rootOrigin, rootBackup, "Regions/Regions.bin");
CopyFile(rootOrigin, rootBackup, "Regions/Regions.idx");
var backupPath = PathUtility.EnsureDirectory(Path.Combine(AutoArchive.BackupPath, "Crashed", timeStamp));
var savePath = World.SavePath;
if (Directory.Exists(savePath))
{
DirectoryCopy(savePath, backupPath);
}
logger.Information("Backup done");
}
@ -160,7 +146,7 @@ namespace Server.Misc
try
{
op.WriteLine("Mobiles: {0}", World.Mobiles.Count);
op.WriteLine($"Accounts: {Accounts.Count}");
}
catch
{
@ -169,15 +155,23 @@ namespace Server.Misc
try
{
op.WriteLine("Items: {0}", World.Items.Count);
op.WriteLine($"Mobiles: {World.Mobiles.Count}");
}
catch
{
// ignored
}
op.WriteLine("Exception:");
op.WriteLine(e.Exception);
try
{
op.WriteLine($"Items: {World.Items.Count}");
}
catch
{
// ignored
}
op.WriteLine($"Exception: {e.Exception}");
op.WriteLine();
op.WriteLine("Clients:");
@ -186,22 +180,22 @@ namespace Server.Misc
{
var states = TcpServer.Instances;
op.WriteLine("- Count: {0}", states.Count);
op.WriteLine($"- Count: {states.Count}");
foreach (var ns in TcpServer.Instances)
{
op.Write("+ {0}:", ns);
op.Write($"+ {ns}:");
if (ns.Account is Account a)
{
op.Write(" (account = {0})", a.Username);
op.Write($" (account = {a.Username})");
}
var m = ns.Mobile;
if (m != null)
{
op.Write(" (mobile = 0x{0:X} '{1}')", m.Serial.Value, m.Name);
op.Write($" (mobile = 0x{m.Serial.Value:X} '{m.Name}')");
}
op.WriteLine();

318
Projects/UOContent/World Saves/AutoArchive.cs
View file

@ -2,9 +2,9 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using Microsoft.Toolkit.HighPerformance;
using Server.Compression;
using Server.Logging;
@ -29,6 +29,7 @@ namespace Server.Saves
{
private static readonly ILogger logger = LogFactory.GetLogger(typeof(AutoArchive));
private static string _tempArchivePath;
private static int _isArchiving;
private static DateTime _nextHourlyArchive;
private static DateTime _nextDailyArchive;
@ -40,16 +41,25 @@ namespace Server.Saves
public static Action Prune { get; set; }
public static string ArchivePath { get; private set; }
public static string BackupPath { get; private set; }
public static string AutomaticBackupPath { get; private set; }
public static void Configure()
{
BackupPath = ServerConfiguration.GetOrUpdateSetting("autoArchive.backupPath", "Backups/Automatic");
ArchivePath = ServerConfiguration.GetOrUpdateSetting("autoArchive.archivePath", "Archives");
_compressionFormat = ServerConfiguration.GetOrUpdateSetting("autoArchive.compressionFormat", CompressionFormat.Zstd);
var tempArchivePath = ServerConfiguration.GetSetting("autoArchive.tempArchivePath", "temp");
_tempArchivePath = PathUtility.GetFullPath(tempArchivePath);
var backupPath = ServerConfiguration.GetOrUpdateSetting("autoArchive.backupPath", "Backups");
BackupPath = PathUtility.GetFullPath(backupPath);
AutomaticBackupPath = Path.Combine(BackupPath, "Automatic");
var archivePath = ServerConfiguration.GetOrUpdateSetting("autoArchive.archivePath", "Archives");
ArchivePath = PathUtility.GetFullPath(archivePath);
var useLocalArchives = ServerConfiguration.GetOrUpdateSetting("autoArchive.archiveLocally", true);
_enablePruning = ServerConfiguration.GetOrUpdateSetting("autoArchive.enableArchivePruning", true);
_compressionFormat = ServerConfiguration.GetOrUpdateSetting("autoArchive.compressionFormat", CompressionFormat.Zstd);
if (useLocalArchives)
{
Archive = AutoArchiveLocally;
@ -85,8 +95,8 @@ namespace Server.Saves
return;
}
var backupPath = Path.Combine(BackupPath, Utility.GetTimeStamp());
AssemblyHandler.EnsureDirectory(BackupPath);
Directory.CreateDirectory(AutomaticBackupPath);
var backupPath = Path.Combine(AutomaticBackupPath, Utility.GetTimeStamp());
Directory.Move(args.OldSavePath, backupPath);
logger.Information($"Created backup at {backupPath}");
@ -97,20 +107,16 @@ namespace Server.Saves
private static void RestoreFromArchive()
{
var savePath = Path.Combine(Core.BaseDirectory, ServerConfiguration.GetSetting("world.savePath", "Saves"));
var files = Directory.Exists(savePath) ? Directory.GetFiles(savePath, "????-??-??-??-??-??.*") : null;
if (files == null || files.Length == 0)
if (!Directory.Exists(savePath))
{
return;
}
var latestFiles = GetInRange(files, DateTime.MinValue, DateTime.MaxValue, true);
foreach (var file in latestFiles)
foreach (var file in PathsByTimestampName(savePath, true))
{
if (RestoreFromFile(file, savePath))
{
File.Delete(file);
break;
}
}
@ -128,19 +134,10 @@ namespace Server.Saves
logger.Information($"Restoring latest world save from archive {fileName}");
var tempFolder = Path.Combine(Core.BaseDirectory, "temp");
AssemblyHandler.EnsureDirectory(tempFolder);
bool successful;
if (fileName.EndsWithOrdinal(".tar.zst"))
{
successful = ZstdArchive.ExtractToDirectory(fi.FullName, tempFolder);
}
else
{
TarArchive.ExtractToDirectory(fi.FullName, tempFolder);
successful = true;
}
var tempPath = PathUtility.EnsureRandomPath(_tempArchivePath);
var successful = fileName.EndsWithOrdinal(".tar.zst")
? ZstdArchive.ExtractToDirectory(fi.FullName, tempPath)
: TarArchive.ExtractToDirectory(fi.FullName, tempPath);
if (!successful)
{
@ -148,25 +145,17 @@ namespace Server.Saves
return false;
}
var allFolders = Directory.EnumerateDirectories(tempFolder, "????-??-??-??-??-??");
var worldSaveFolders = GetInRange(allFolders, DateTime.MinValue, DateTime.MaxValue);
var first = true;
foreach (var folder in worldSaveFolders)
foreach (var folder in PathsByTimestampName(tempPath))
{
if (first)
{
first = false;
Directory.Delete(savePath, true);
var dirInfo = new DirectoryInfo(folder);
logger.Information($"Restoring backup {dirInfo.Name}");
Directory.Move(folder, savePath);
}
else
{
Directory.Delete(folder, true);
}
Directory.Delete(savePath, true);
var dirInfo = new DirectoryInfo(folder);
logger.Information($"Restoring backup {dirInfo.Name}");
Directory.Move(folder, savePath);
break;
}
Directory.Delete(tempPath, true);
return true;
}
@ -180,14 +169,26 @@ namespace Server.Saves
PruneLocalArchives(ArchivePeriod.Monthly, 12);
}
if (Directory.Exists(BackupPath))
if (!Directory.Exists(AutomaticBackupPath))
{
var allFolders = Directory.EnumerateDirectories(BackupPath, "????-??-??-??-??-??", SearchOption.AllDirectories);
var latestBackupFolders = GetInRange(allFolders, DateTime.MinValue, Core.Now.AddMonths(-1));
return;
}
foreach (var folder in latestBackupFolders)
var allFolders = Directory.EnumerateDirectories(AutomaticBackupPath);
var threshold = Core.Now.AddMonths(-1);
foreach (var folder in allFolders)
{
var dirName = new DirectoryInfo(folder).Name;
if (!TryGetDate(dirName, out var date))
{
logger.Information($"Pruning backup {folder}");
continue;
}
if (date < threshold)
{
logger.Information($"Pruning old backup {folder}");
Directory.Delete(folder, true);
}
}
@ -196,12 +197,14 @@ namespace Server.Saves
private static void PruneLocalArchives(ArchivePeriod period, int minRetained)
{
var periodStr = period.ToString();
var path = Path.Combine(ArchivePath, periodStr);
var allFiles = Directory.EnumerateFiles(path, "????-??-??-??-??-??.*");
var archives = GetInRange(allFiles, DateTime.MinValue, DateTime.MaxValue, true);
var archivePath = Path.Combine(ArchivePath, periodStr);
if (!Directory.Exists(archivePath))
{
return;
}
var periodLowerStr = periodStr.ToLowerInvariant();
foreach (var archive in archives)
foreach (var archive in PathsByTimestampName(archivePath, true))
{
if (minRetained > 0)
{
@ -234,98 +237,150 @@ namespace Server.Saves
{
if (date >= _nextHourlyArchive)
{
var lastHour = date.Date.AddHours(date.Hour);
Rollup(ArchivePeriod.Hourly, lastHour.AddHours(-1), lastHour.ToTimeStamp());
_nextHourlyArchive = _nextHourlyArchive.AddHours(2);
Rollup(ArchivePeriod.Hourly);
_nextHourlyArchive = _nextHourlyArchive.AddHours(1);
}
if (date >= _nextDailyArchive)
{
var yesterday = date.Date.AddDays(-1);
Rollup(ArchivePeriod.Daily, yesterday, yesterday.ToTimeStamp());
_nextDailyArchive = _nextDailyArchive.AddDays(2);
Rollup(ArchivePeriod.Daily);
_nextDailyArchive = _nextDailyArchive.AddDays(1);
}
if (date >= _nextMonthlyArchive)
{
var lastMonth = new DateTime(date.Year, date.Month, 1).AddMonths(-1);
Rollup(ArchivePeriod.Monthly, lastMonth, lastMonth.ToTimeStamp());
_nextMonthlyArchive = _nextMonthlyArchive.AddMonths(2);
Rollup(ArchivePeriod.Monthly);
_nextMonthlyArchive = _nextMonthlyArchive.AddMonths(1);
}
Prune?.Invoke();
_isArchiving = 0;
},
null
);
}
private static void Rollup(ArchivePeriod archivePeriod, DateTime rangeStart, string archiveNameNoExtension)
private static void Rollup(ArchivePeriod archivePeriod)
{
if (!Directory.Exists(BackupPath))
if (!Directory.Exists(AutomaticBackupPath))
{
return;
}
var currentRangeStart = Core.Now.ArchivePeriodStart(archivePeriod);
var allFolders = Directory.EnumerateDirectories(AutomaticBackupPath);
var items = new Dictionary<DateTime, SortedDictionary<DateTime, string>>();
foreach (var path in allFolders)
{
var dirName = new DirectoryInfo(path).Name;
if (!TryGetDate(dirName, out var date))
{
continue;
}
var rangeStart = date.ArchivePeriodStart(archivePeriod);
// Only archive the past
if (rangeStart >= currentRangeStart)
{
continue;
}
if (items.TryGetValue(rangeStart, out var value))
{
value.Add(date, path);
}
else
{
value = new SortedDictionary<DateTime, string>(new DescendingComparer<DateTime>())
{
{ date, path }
};
items.Add(rangeStart, value);
}
}
var archivePeriodStr = archivePeriod.ToString();
var archivePath = PathUtility.EnsureDirectory(Path.Combine(ArchivePath, archivePeriodStr));
var archivePeriodStrLower = archivePeriodStr.ToLowerInvariant();
var stopWatch = new Stopwatch();
stopWatch.Start();
var allFolders = Directory.EnumerateDirectories(BackupPath, "????-??-??-??-??-??");
var rangeEnd = archivePeriod switch
{
ArchivePeriod.Monthly => rangeStart.AddMonths(1),
ArchivePeriod.Daily => rangeStart.AddDays(1),
_ => rangeStart.AddHours(1),
};
var latestFolders = GetInRange(allFolders, rangeStart, rangeEnd).ToList();
// We don't want to re-archive a single save. This usually means we rebooted and found the one we purposefully left behind
if (latestFolders.Count <= 1)
{
return;
}
var archivePath = Path.Combine(ArchivePath, archivePeriodStr);
AssemblyHandler.EnsureDirectory(archivePath);
var extension = _compressionFormat.GetFileExtension();
var archiveFilePath = Path.Combine(archivePath, $"{archiveNameNoExtension}{extension}");
logger.Information($"Creating {archivePeriodStrLower} archive");
var archiveCreated = _compressionFormat == CompressionFormat.Zstd
? ZstdArchive.CreateFromPaths(latestFolders, archiveFilePath)
: TarArchive.CreateFromPaths(latestFolders, archiveFilePath);
// Leave behind 1 hourly for daily, 1 daily for monthly.
var minimum = archivePeriod != ArchivePeriod.Monthly ? 1 : 0;
if (archiveCreated)
foreach (var (rangeStart, sortedBackups) in items)
{
stopWatch.Stop();
var elapsed = stopWatch.Elapsed.TotalSeconds;
logger.Information($"Created {archivePeriodStrLower} archive at {archiveFilePath} ({elapsed:F2} seconds)");
// Keep the latest one, but delete the rest.
for (var i = 1; i < latestFolders.Count; i++)
var backups = sortedBackups.Values;
if (backups.Count == 0)
{
Directory.Delete(latestFolders[i], true);
continue;
}
}
else
{
logger.Warning($"Failed to create {archivePeriodStrLower} archive");
var stopWatch = new Stopwatch();
stopWatch.Start();
var fileName = $"{rangeStart.ToTimeStamp(archivePeriod)}{extension}";
var archiveFilePath = Path.Combine(archivePath, fileName);
var archiveCreated = CreateArchive(archiveFilePath, AutomaticBackupPath, backups);
if (archiveCreated)
{
var elapsed = stopWatch.Elapsed.TotalSeconds;
logger.Information($"Created {archivePeriodStrLower} archive at {archiveFilePath} ({elapsed:F2} seconds)");
var i = minimum;
foreach (var backup in backups)
{
// Keep the latest one, but delete the rest.
if (i-- > 0)
{
continue;
}
Directory.Delete(backup, true);
}
}
else
{
logger.Warning($"Failed to create {archivePeriodStrLower} archive");
}
stopWatch.Stop();
}
}
private static IEnumerable<string> GetInRange(
IEnumerable<string> allItems,
DateTime rangeStart,
DateTime rangeEnd,
bool files = false
)
private static bool CreateArchive(string archiveFilePath, string relativeTo, IEnumerable<string> backups) =>
_compressionFormat == CompressionFormat.Zstd
? ZstdArchive.CreateFromPaths(backups, archiveFilePath, relativeTo)
: TarArchive.CreateFromPaths(backups, archiveFilePath, relativeTo);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static string ToTimeStamp(this DateTime date, ArchivePeriod archivePeriod) =>
archivePeriod switch
{
ArchivePeriod.Monthly => date.ToString("yyyy-MM"),
ArchivePeriod.Daily => date.ToString("yyyy-MM-dd"),
ArchivePeriod.Hourly => date.ToString("yyyy-MM-dd-HH"),
_ => date.ToTimeStamp()
};
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static DateTime ArchivePeriodStart(this DateTime date, ArchivePeriod archivePeriod) =>
archivePeriod switch
{
ArchivePeriod.Monthly => date.Date.AddDays(-(date.Day - 1)),
ArchivePeriod.Daily => date.Date,
ArchivePeriod.Hourly => date.Date.AddHours(date.Hour),
_ => date
};
private static IEnumerable<string> PathsByTimestampName(string path, bool files = false)
{
var allItems = files ? Directory.GetFiles(path) : Directory.GetDirectories(path);
var items = new SortedDictionary<DateTime, string>(new DescendingComparer<DateTime>());
foreach (var item in allItems)
{
@ -333,14 +388,14 @@ namespace Server.Saves
if (files)
{
var fileName = new FileInfo(item).Name;
name = fileName[..fileName.IndexOfOrdinal(".")];
name = fileName[..fileName.IndexOf('.')];
}
else
{
name = new DirectoryInfo(item).Name;
}
if (IsInRange(name, rangeStart, rangeEnd, out var date))
if (TryGetDate(name, out var date))
{
items.Add(date, item);
}
@ -349,38 +404,41 @@ namespace Server.Saves
return items.Values;
}
private static bool IsInRange(string name, DateTime start, DateTime end, out DateTime date) =>
TryGetDate(name, out date) && start <= date && end >= date;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryGetDate(string value, out DateTime date)
{
// Expecting YYYY-MM-DD-HH-mm-ss
var split = value.Split("-");
if (split.Length != 6)
{
date = DateTime.MinValue;
return false;
}
Span<int> parts = stackalloc int[] { 0, 1, 1, 0, 0, 0 };
try
{
var i = 0;
foreach (var part in value.Tokenize('-'))
{
parts[i++] = int.Parse(part);
}
if (i == 0)
{
date = DateTime.MinValue;
return false;
}
date = new DateTime(
int.Parse(split[0]),
int.Parse(split[1]),
int.Parse(split[2]),
int.Parse(split[3]),
int.Parse(split[4]),
int.Parse(split[5])
parts[0],
parts[1],
parts[2],
parts[3],
parts[4],
parts[5],
DateTimeKind.Utc
);
return true;
}
catch
{
logger.Warning($"Path was not in the correct date format: {value}");
date = DateTime.MinValue;
return false;
}
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]