feat: Adds auto archiving (#794)
## Adds Auto Archiving Backups are archived once an hour, day, and month. Archives older than 60 days are pruned automatically. _Note: Automatic pruning is off by default_ ### Archive compression format The following formats are supported: * Zstd (The fastest with best compression ratio) * GZip * Zip * None (Tar) _Note: By default archives use [zstandard](http://facebook.github.io/zstd/) format. The archive format can be changed in modernuo.json `autoArchive.compressionFormat`_ ### Restoring world from archive Move the archive to the Saves folder (tar.zst file). On startup the server will extract the file and restore the latest save. See pictures below. ### Manually extracting an archives #### Windows * Use the latest version of [7-zip w/ ZStandard](https://github.com/mcmilk/7-Zip-zstd/releases/latest) 1. Extract the `.tar.zst` file. 2. Extract the `.tar` file. (Yes you have to do it in two steps) * On Windows 10 you can use the command line. `zstd.exe` is in the Assemblies folder after building ModernUO. 1. `tar --use-compress-program "Distribution\Assemblies\zstd.exe -d" -xvf "Archives\Hourly\archivefile.tar.zst" -C "path to where you want to extract it"` #### Mac * Install [Keka](https://www.keka.io) 1. Drop the .tar.zst onto the keka interface. #### Linux 1. Install zstd from a package manager 2. Run `tar -I zstd -xvf "Archives\Hourly\archivefile.tar.zst" -C "path to where you want to extract it"` ### Other changes * Changes Autosave to occur at the same time no matter when the server is booted. * Adds `[SaveFrequency <delay> [warning]`command to set save frequency and warning frequency in-game. * Adds support for time zones that are configurable. The system timezone can also be manually configured. Check `TimeZoneHandler.cs` for details. <img width="257" alt="Screen Shot 2021-09-22 at 11 23 18 PM" src="https://user-images.githubusercontent.com/3953314/134464929-a5bf3cd8-2ef0-4476-a9ad-71d0816a19ac.png"> <img width="241" alt="Screen Shot 2021-09-22 at 11 23 39 PM" src="https://user-images.githubusercontent.com/3953314/134464945-5f6f96dc-3d1e-434d-8256-5c6b3705e786.png"> <img width="836" alt="Screen Shot 2021-09-22 at 11 34 39 PM" src="https://user-images.githubusercontent.com/3953314/134464957-625e57f2-ef47-4ca1-a0a7-cd40c9b6539c.png"> <img width="631" alt="Screen Shot 2021-09-22 at 11 34 50 PM" src="https://user-images.githubusercontent.com/3953314/134464969-b2d65c0d-f8f5-497a-a832-5d805e8e0b22.png">
This commit is contained in:
parent
2f5b26ae68
commit
3f6a87483b
22 changed files with 2150 additions and 149 deletions
113
Projects/UOContent/Compression/TarArchive.cs
Executable file
113
Projects/UOContent/Compression/TarArchive.cs
Executable file
|
|
@ -0,0 +1,113 @@
|
|||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using Server.Buffers;
|
||||
|
||||
namespace Server.Compression
|
||||
{
|
||||
public static class TarArchive
|
||||
{
|
||||
private const string _libArchiveWindowsUrl = @"https://www.libarchive.org/downloads/libarchive-v3.5.2-win64.zip";
|
||||
private static string _pathToTar;
|
||||
|
||||
private static string GetPathToTar()
|
||||
{
|
||||
if (!Core.IsWindows || File.Exists(@"C:\Windows\system32\tar.exe"))
|
||||
{
|
||||
return "tar";
|
||||
}
|
||||
|
||||
var tarPath = Path.Combine(Core.BaseDirectory, "bsdtar/bsdtar.exe");
|
||||
return File.Exists(tarPath) ? tarPath : DownloadTarForWindows();
|
||||
}
|
||||
|
||||
private static int RunTar(string arguments, string compressionProgramPath)
|
||||
{
|
||||
var process = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = _pathToTar,
|
||||
Arguments = arguments
|
||||
}
|
||||
};
|
||||
|
||||
if (compressionProgramPath != null)
|
||||
{
|
||||
var envPaths = $"{process.StartInfo.EnvironmentVariables["PATH"]}:{compressionProgramPath}";
|
||||
process.StartInfo.EnvironmentVariables["PATH"] = envPaths;
|
||||
}
|
||||
|
||||
process.Start();
|
||||
process.WaitForExit();
|
||||
|
||||
return process.ExitCode;
|
||||
}
|
||||
|
||||
private static string DownloadTarForWindows()
|
||||
{
|
||||
var tempDir = Path.Combine(Core.BaseDirectory, "temp");
|
||||
AssemblyHandler.EnsureDirectory("temp");
|
||||
|
||||
var libarchiveFile = Path.Combine(tempDir, "libarchive.zip");
|
||||
using WebClient wc = new WebClient();
|
||||
wc.DownloadFile (new Uri(_libArchiveWindowsUrl), libarchiveFile);
|
||||
|
||||
ZipFile.ExtractToDirectory(libarchiveFile, tempDir);
|
||||
var libArchivePath = Path.Combine(tempDir, "libarchive");
|
||||
Directory.Move(Path.Combine(libArchivePath, "bin"), "bsdtar");
|
||||
Directory.Delete(libArchivePath, true);
|
||||
File.Delete(libarchiveFile);
|
||||
|
||||
return Path.Combine(Core.BaseDirectory, "bsdtar/bsdtar.exe");
|
||||
}
|
||||
|
||||
public static bool ExtractToDirectory(
|
||||
string fileNamePath,
|
||||
string outputDirectory,
|
||||
string compressCommand = null,
|
||||
string compressionProgramPath = null
|
||||
)
|
||||
{
|
||||
_pathToTar ??= GetPathToTar();
|
||||
|
||||
var tarFlags = compressCommand == null ? "-axf" : "-xf";
|
||||
var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : "";
|
||||
var arguments = $"{useExternalCompression}{tarFlags} \"{fileNamePath}\" -C \"{outputDirectory}\"";
|
||||
|
||||
return RunTar(arguments, compressionProgramPath) == 0;
|
||||
}
|
||||
|
||||
public static bool CreateFromPaths(
|
||||
List<string> paths,
|
||||
string destinationArchiveFileName,
|
||||
string compressCommand = null,
|
||||
string compressionProgramPath = null
|
||||
)
|
||||
{
|
||||
_pathToTar ??= GetPathToTar();
|
||||
|
||||
AssemblyHandler.EnsureDirectory(new FileInfo(destinationArchiveFileName));
|
||||
var di = new DirectoryInfo(paths[0]);
|
||||
var directory = di.Parent!.FullName;
|
||||
|
||||
using var builder = new ValueStringBuilder();
|
||||
for (var i = 0; i < paths.Count; i++)
|
||||
{
|
||||
var path = paths[i];
|
||||
builder.Append($"{(i > 0 ? " " : "")}\"{Path.GetRelativePath(directory, path)}\"");
|
||||
}
|
||||
var pathsToCompress = builder.ToString();
|
||||
|
||||
var tarFlags = compressCommand == null ? "-acf" : "-cf";
|
||||
var useExternalCompression = compressCommand != null ? $"--use-compress-program \"{compressCommand}\" " : "";
|
||||
var arguments = $"{useExternalCompression}{tarFlags} \"{destinationArchiveFileName}\" -C \"{directory}\" {pathsToCompress}";
|
||||
|
||||
return RunTar(arguments, compressionProgramPath) == 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
95
Projects/UOContent/Compression/ZstdArchive.cs
Executable file
95
Projects/UOContent/Compression/ZstdArchive.cs
Executable file
|
|
@ -0,0 +1,95 @@
|
|||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
|
||||
namespace Server.Compression
|
||||
{
|
||||
public static class ZstdArchive
|
||||
{
|
||||
private static readonly string _pathToZstd = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory?.FullName;
|
||||
|
||||
public static bool ExtractToDirectory(string fileNamePath, string outputDirectory)
|
||||
{
|
||||
// bsdtar has a bug and hangs, so we are doing it in two steps.
|
||||
if (Core.IsWindows)
|
||||
{
|
||||
var tempTarArchive = Path.Combine(Core.BaseDirectory, "temp/temp-file.tar");
|
||||
|
||||
try
|
||||
{
|
||||
var process = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = _pathToZstd,
|
||||
Arguments = $"--no-progress -d \"{fileNamePath}\" -o \"${tempTarArchive}\""
|
||||
}
|
||||
};
|
||||
|
||||
process.Start();
|
||||
process.WaitForExit();
|
||||
|
||||
return process.ExitCode == 0 && TarArchive.ExtractToDirectory(tempTarArchive, outputDirectory);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(tempTarArchive);
|
||||
}
|
||||
}
|
||||
|
||||
return TarArchive.ExtractToDirectory(fileNamePath, outputDirectory, "zstd -d", _pathToZstd);
|
||||
}
|
||||
|
||||
public static bool CreateFromPaths(
|
||||
List<string> paths,
|
||||
string destinationArchiveFileName,
|
||||
int compressionLevel = 10
|
||||
)
|
||||
{
|
||||
Debug.Assert(compressionLevel is >= 1 and <= 22, $"{nameof(compressionLevel)} must be between 1 and 22");
|
||||
|
||||
// bsdtar has a bug and hangs, so we are doing it in two steps.
|
||||
if (Core.IsWindows)
|
||||
{
|
||||
var tempTarArchive = Path.Combine(Core.BaseDirectory, "temp/temp-file.tar");
|
||||
|
||||
try
|
||||
{
|
||||
if (!TarArchive.CreateFromPaths(paths, tempTarArchive))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var process = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = $"{_pathToZstd}\\zstd.exe",
|
||||
Arguments = $"--no-progress -10 \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\""
|
||||
}
|
||||
};
|
||||
|
||||
process.Start();
|
||||
process.WaitForExit();
|
||||
|
||||
return process.ExitCode == 0;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(tempTarArchive);
|
||||
}
|
||||
}
|
||||
|
||||
return TarArchive.CreateFromPaths(paths, destinationArchiveFileName, $"zstd -10", _pathToZstd);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue