feat: Updates serialization to use MMF (considerable memory savings) (#1841)

### Summary
Updates the serialization strategy to use `MemoryMappedFile` instead of thick buffers. This has the benefit of being on-par with the current implementation (based on hardware/OS), however won't incur the double-memory issue.

> [!Important]
> **Developer Note**
> The `BinaryFileWriter` and `BinaryFileReader` has been removed in favor of `MemoryMapFileWriter` and `UnmanagedDataReader`
This commit is contained in:
Kamron Batman 2024-06-23 10:51:20 -07:00 • committed by GitHub
parent 3b84c8052c
commit 8ec203f387
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 1271 additions and 941 deletions

View file

@ -1,330 +0,0 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2023 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: EntityPersistence.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.MemoryMappedFiles;
using System.Reflection;
namespace Server;
public static class EntityPersistence
{
public static void WriteEntities<I, T>(
IIndexInfo<I> indexInfo,
Dictionary<I, T> entities,
string savePath,
ConcurrentQueue<Type> types
) where T : class, ISerializable
{
var typeName = indexInfo.TypeName;
var path = Path.Combine(savePath, typeName);
PathUtility.EnsureDirectory(path);
string idxPath = Path.Combine(path, $"{typeName}.idx");
string binPath = Path.Combine(path, $"{typeName}.bin");
using var idx = new BinaryFileWriter(idxPath, false, types);
using var bin = new BinaryFileWriter(binPath, true, types);
idx.Write(3); // Version
idx.Write(entities.Count);
foreach (var e in entities.Values)
{
long start = bin.Position;
var t = e.GetType();
idx.Write(t);
idx.Write(e.Serial);
idx.Write(e.Created.Ticks);
idx.Write(start);
e.SerializeTo(bin);
idx.Write((int)(bin.Position - start));
}
}
public static Dictionary<I, T> LoadIndex<I, T>(
string path,
IIndexInfo<I> indexInfo,
Dictionary<ulong, string> serializedTypes,
out List<EntitySpan<T>> entities
) where T : class, ISerializable
{
var map = new Dictionary<I, T>();
object[] ctorArgs = new object[1];
var indexType = indexInfo.TypeName;
string indexPath = Path.Combine(path, indexType, $"{indexType}.idx");
entities = new List<EntitySpan<T>>();
if (!File.Exists(indexPath))
{
return map;
}
using FileStream idx = new FileStream(indexPath, FileMode.Open, FileAccess.Read, FileShare.Read);
BinaryReader idxReader = new BinaryReader(idx);
var version = idxReader.ReadInt32();
int count = idxReader.ReadInt32();
var ctorArguments = new[] { typeof(I) };
List<ConstructorInfo> types;
string typesPath = Path.Combine(path, indexType, $"{indexType}.tdb");
if (File.Exists(typesPath))
{
using FileStream tdb = new FileStream(typesPath, FileMode.Open, FileAccess.Read, FileShare.Read);
BinaryReader tdbReader = new BinaryReader(tdb);
types = ReadTypes(tdbReader, ctorArguments);
tdbReader.Close();
}
else
{
types = null;
}
// We must have a typeDb from SerializedTypes.db, or a tdb file
if (serializedTypes == null && types == null)
{
return map;
}
var now = DateTime.UtcNow;
for (int i = 0; i < count; ++i)
{
ConstructorInfo ctor;
if (version >= 2)
{
var flag = idxReader.ReadByte();
if (flag != 2)
{
throw new Exception($"Invalid type flag, expected 2 but received {flag}.");
}
var hash = idxReader.ReadUInt64();
serializedTypes!.TryGetValue(hash, out var typeName);
ctor = GetConstructorFor(typeName, AssemblyHandler.FindTypeByHash(hash), ctorArguments);
}
else
{
ctor = types?[idxReader.ReadInt32()];
}
var serial = idxReader.ReadUInt32();
var created = version == 0 ? now : new DateTime(idxReader.ReadInt64(), DateTimeKind.Utc);
if (version is > 0 and < 3)
{
idxReader.ReadInt64(); // LastSerialized
}
var pos = idxReader.ReadInt64();
var length = idxReader.ReadInt32();
if (ctor == null)
{
continue;
}
I indexer = indexInfo.CreateIndex(serial);
ctorArgs[0] = indexer;
if (ctor.Invoke(ctorArgs) is T entity)
{
entity.Created = created;
entities.Add(new EntitySpan<T>(entity, pos, length));
map[indexer] = entity;
}
}
idxReader.Close();
entities.TrimExcess();
return map;
}
public static void LoadData<I, T>(
string path,
IIndexInfo<I> indexInfo,
Dictionary<ulong, string> serializedTypes,
List<EntitySpan<T>> entities
) where T : class, ISerializable
{
var indexType = indexInfo.TypeName;
string dataPath = Path.Combine(path, indexType, $"{indexType}.bin");
if (!File.Exists(dataPath) || new FileInfo(dataPath).Length == 0)
{
return;
}
using var mmf = MemoryMappedFile.CreateFromFile(dataPath, FileMode.Open);
using var stream = mmf.CreateViewStream();
BufferReader br = null;
var deleteAllFailures = false;
foreach (var entry in entities)
{
T t = entry.Entity;
var position = entry.Position;
stream.Seek(position, SeekOrigin.Begin);
// Skip this entry
if (t == null)
{
continue;
}
if (entry.Length == 0)
{
t.Delete();
continue;
}
var buffer = GC.AllocateUninitializedArray<byte>(entry.Length);
if (br == null)
{
br = new BufferReader(buffer, serializedTypes);
}
else
{
br.Reset(buffer, out _);
}
stream.Read(buffer.AsSpan());
string error;
try
{
t.Deserialize(br);
error = br.Position != entry.Length
? $"Serialized object was {entry.Length} bytes, but {br.Position} bytes deserialized"
: null;
}
catch (Exception e)
{
error = e.ToString();
}
if (error == null)
{
t.InitializeSaveBuffer(buffer, World.SerializedTypes);
}
else
{
Console.WriteLine($"***** Bad deserialize of {t.GetType()} ({t.Serial}) *****");
Console.WriteLine(error);
if (!deleteAllFailures)
{
Console.Write("Delete the object and continue? (y/n/a): ");
var pressedKey = Console.ReadLine();
if (pressedKey.InsensitiveEquals("a"))
{
deleteAllFailures = true;
}
else if (!pressedKey.InsensitiveEquals("y"))
{
throw new Exception("Deserialization failed.");
}
}
t.Delete();
}
}
}
private static ConstructorInfo GetConstructorFor(string typeName, Type t, Type[] constructorTypes)
{
if (t?.IsAbstract != false)
{
Console.WriteLine("failed");
var issue = t?.IsAbstract == true ? "marked abstract" : "not found";
Console.Write($"Error: Type '{typeName}' was {issue}. Delete all of those types? (y/n): ");
if (Console.ReadLine().InsensitiveEquals("y"))
{
Console.WriteLine("Loading...");
return null;
}
Console.WriteLine("Types will not be deleted. An exception will be thrown.");
throw new Exception($"Bad type '{typeName}'");
}
var ctor = t.GetConstructor(constructorTypes);
if (ctor == null)
{
throw new Exception($"Type '{t}' does not have a serialization constructor");
}
return ctor;
}
/**
* Legacy ReadTypes for backward compatibility with old saves that still have a tdb file
*/
private static List<ConstructorInfo> ReadTypes(BinaryReader tdbReader, Type[] ctorArguments)
{
var count = tdbReader.ReadInt32();
var types = new List<ConstructorInfo>(count);
for (var i = 0; i < count; ++i)
{
var typeName = tdbReader.ReadString();
var t = AssemblyHandler.FindTypeByFullName(typeName, false);
var ctor = GetConstructorFor(typeName, t, ctorArguments);
types.Add(ctor);
}
return types;
}
private static void SerializeTo(this ISerializable entity, IGenericWriter writer)
{
var saveBuffer = entity.SaveBuffer;
// If nothing was serialized we expect the object to be deleted on deserialization
if (saveBuffer.Position == 0)
{
return;
}
// Resize to the exact size
saveBuffer.Resize((int)saveBuffer.Position);
// Write that amount
writer.Write(saveBuffer.Buffer);
}
}

View file

@ -1,8 +1,8 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2023 - ModernUO Development Team *
* Copyright 2019-2024 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: IWorldSerializable.cs *
* File: IGenericSerializable.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
@ -13,12 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
using System;
using System.Collections.Concurrent;
namespace Server;
public interface IGenericSerializable
{
void Serialize(ConcurrentQueue<Type> types);
void Serialize(IGenericWriter writer);
}

View file

@ -1,6 +1,6 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2023 - ModernUO Development Team *
* Copyright 2019-2024 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: World.cs *
* *
@ -38,7 +38,7 @@ public enum WorldState
public static class World
{
private static ILogger logger = LogFactory.GetLogger(typeof(World));
private static readonly ILogger logger = LogFactory.GetLogger(typeof(World));
private static readonly ItemPersistence _itemPersistence = new();
private static readonly MobilePersistence _mobilePersistence = new();
@ -185,68 +185,10 @@ public static class World
// Create the serialization threads.
for (var i = 0; i < _threadWorkers.Length; i++)
{
_threadWorkers[i] = new SerializationThreadWorker();
_threadWorkers[i] = new SerializationThreadWorker(i);
}
}
private static void FinishWorldSave()
{
WorldState = WorldState.Running;
Persistence.PostSerializeAll(); // Process safety queues
}
public static void WriteFiles(object state)
{
Exception exception = null;
var tempPath = PathUtility.EnsureRandomPath(_tempSavePath);
try
{
var watch = Stopwatch.StartNew();
logger.Information("Writing world save snapshot");
Persistence.WriteSnapshot(tempPath, SerializedTypes);
watch.Stop();
logger.Information("Writing world save snapshot {Status} ({Duration:F2} seconds)", "done", watch.Elapsed.TotalSeconds);
}
catch (Exception ex)
{
exception = ex;
}
if (exception != null)
{
logger.Error(exception, "Writing world save snapshot {Status}.", "failed");
Persistence.TraceException(exception);
BroadcastStaff(0x35, true, "Writing world save snapshot failed.");
}
else
{
try
{
EventSink.InvokeWorldSavePostSnapshot(SavePath, tempPath);
PathUtility.MoveDirectoryContents(tempPath, SavePath);
Directory.SetLastWriteTimeUtc(SavePath, Core.Now);
}
catch (Exception ex)
{
Persistence.TraceException(ex);
}
}
// Clear types
SerializedTypes.Clear();
_diskWriteHandle.Set();
Core.LoopContext.Post(FinishWorldSave);
}
private static void ProcessDecay()
{
while (_decayQueue.TryDequeue(out var item))
@ -301,28 +243,39 @@ public static class World
return;
}
WaitForWriteCompletion(); // Blocks Save until current disk flush is done.
_diskWriteHandle.Reset();
// Start our serialization threads
for (var i = 0; i < _threadWorkers.Length; i++)
{
_threadWorkers[i].Wake();
}
WorldState = WorldState.PendingSave;
Core.RequestSnapshot();
ThreadPool.QueueUserWorkItem(Preserialize);
}
internal static TimeSpan Snapshot()
internal static void Preserialize(object state)
{
var tempPath = PathUtility.EnsureRandomPath(_tempSavePath);
try
{
Persistence.PreSerializeAll(tempPath, SerializedTypes);
Core.RequestSnapshot(tempPath);
}
catch (Exception ex)
{
logger.Error(ex, "Preparing to save world {Status}", "failed");
Persistence.TraceException(ex);
BroadcastStaff(0x35, true, "Preparing for a world save failed! Check the logs!");
}
}
internal static void Snapshot(string snapshotPath)
{
if (WorldState != WorldState.PendingSave)
{
return TimeSpan.Zero;
return;
}
WaitForWriteCompletion(); // Blocks Save until current disk flush is done.
_diskWriteHandle.Reset();
WorldState = WorldState.Saving;
Broadcast(0x35, true, "The world is saving, please wait.");
@ -336,14 +289,10 @@ public static class World
try
{
_serializationStart = Core.Now;
WakeSerializationThreads();
Persistence.SerializeAll();
// Pause the workers
foreach (var worker in _threadWorkers)
{
worker.Sleep();
}
PauseSerializationThreads();
EventSink.InvokeWorldSave();
}
catch (Exception ex)
@ -351,7 +300,10 @@ public static class World
exception = ex;
}
WorldState = WorldState.WritingSave;
WorldState = WorldState.PendingSave;
ThreadPool.QueueUserWorkItem(WriteFiles, snapshotPath);
Persistence.PostWorldSaveAll(); // Process safety queues
watch.Stop();
if (exception == null)
{
@ -367,18 +319,74 @@ public static class World
BroadcastStaff(0x35, true, "World save failed! Check the logs!");
}
}
ThreadPool.QueueUserWorkItem(WriteFiles);
private static void WriteFiles(object state)
{
var snapshotPath = (string)state;
try
{
var watch = Stopwatch.StartNew();
logger.Information("Writing world save snapshot");
Persistence.WriteSnapshotAll(snapshotPath, SerializedTypes);
watch.Stop();
try
{
EventSink.InvokeWorldSavePostSnapshot(SavePath, snapshotPath);
PathUtility.MoveDirectoryContents(snapshotPath, SavePath);
Directory.SetLastWriteTimeUtc(SavePath, Core.Now);
}
catch (Exception ex)
{
Persistence.TraceException(ex);
}
return watch.Elapsed;
watch.Stop();
logger.Information("Writing world save snapshot {Status} ({Duration:F2} seconds)", "done", watch.Elapsed.TotalSeconds);
}
catch (Exception ex)
{
logger.Error(ex, "Writing world save snapshot failed");
Persistence.TraceException(ex);
BroadcastStaff(0x35, true, "Writing world save snapshot failed! Check the logs!");
}
// Clear types
SerializedTypes.Clear();
_diskWriteHandle.Set();
WorldState = WorldState.Running;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void PushToCache(IGenericSerializable e)
internal static void WakeSerializationThreads()
{
_threadWorkers[_threadId++].Push(e);
for (var i = 0; i < _threadWorkers.Length; i++)
{
_threadWorkers[i].Wake();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void PauseSerializationThreads()
{
for (var i = 0; i < _threadWorkers.Length; i++)
{
_threadWorkers[i].Sleep();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static int GetThreadWorkerCount() => Math.Max(Environment.ProcessorCount - 1, 1);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void ResetRoundRobin() => _threadId = 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void PushToCache((IGenericSerializable e, Persistence p) ep)
{
_threadWorkers[_threadId++].Push(ep);
if (_threadId == _threadWorkers.Length)
{
_threadId = 0;
@ -488,15 +496,15 @@ public static class World
EnqueueForDecay(item);
}
PushToCache(item);
PushToCache((item, this));
}
}
public override void PostSerialize()
public override void PostWorldSave()
{
ProcessDecay(); // Run this before the safety queue
base.PostSerialize();
base.PostWorldSave();
}
}
@ -522,18 +530,20 @@ public static class World
private class SerializationThreadWorker
{
private readonly int _index;
private readonly Thread _thread;
private readonly AutoResetEvent _startEvent; // Main thread tells the thread to start working
private readonly AutoResetEvent _stopEvent; // Main thread waits for the worker finish draining
private bool _pause;
private bool _exit;
private readonly ConcurrentQueue<IGenericSerializable> _entities;
private readonly ConcurrentQueue<(IGenericSerializable, Persistence)> _entities;
public SerializationThreadWorker()
public SerializationThreadWorker(int index)
{
_index = index;
_startEvent = new AutoResetEvent(false);
_stopEvent = new AutoResetEvent(false);
_entities = new ConcurrentQueue<IGenericSerializable>();
_entities = new ConcurrentQueue<(IGenericSerializable, Persistence)>();
_thread = new Thread(Execute);
_thread.Start(this);
}
@ -556,14 +566,11 @@ public static class World
Sleep();
}
public void Push(IGenericSerializable entity)
{
_entities.Enqueue(entity);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Push((IGenericSerializable e, Persistence p) ep) => _entities.Enqueue(ep);
private static void Execute(object obj)
{
var serializedTypes = SerializedTypes;
SerializationThreadWorker worker = (SerializationThreadWorker)obj;
var reader = worker._entities;
@ -573,9 +580,10 @@ public static class World
while (true)
{
bool pauseRequested = Volatile.Read(ref worker._pause);
if (reader.TryDequeue(out var entity))
if (reader.TryDequeue(out var ep))
{
entity.Serialize(serializedTypes);
var (e, p) = ep;
p.Serialize(e, worker._index);
}
else if (pauseRequested) // Break when finished
{