warrentyexpired/ModernUO
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: Updates serialization to use MMF (considerable memory savings) (#1841)

Browse source 
### Summary
Updates the serialization strategy to use `MemoryMappedFile` instead of thick buffers. This has the benefit of being on-par with the current implementation (based on hardware/OS), however won't incur the double-memory issue.

> [!Important]
> **Developer Note**
> The `BinaryFileWriter` and `BinaryFileReader` has been removed in favor of `MemoryMapFileWriter` and `UnmanagedDataReader`
This commit is contained in:
Kamron Batman 2024-06-23 10:51:20 -07:00 • committed by GitHub
parent 3b84c8052c
commit 8ec203f387
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 1271 additions and 941 deletions
Download patch file Download diff file Expand all files Collapse all files

106
Projects/Server/Serialization/Persistence.cs
View file

@ -1,6 +1,6 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2023 - ModernUO Development Team *
* Copyright 2019-2024 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: Persistence.cs *
* *
@ -17,6 +17,7 @@ using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.MemoryMappedFiles;
namespace Server;
@ -52,61 +53,54 @@ public abstract class Persistence
}
}
private static Dictionary<ulong, string> LoadTypes(string path)
private unsafe static Dictionary<ulong, string> LoadTypes(string path)
{
var db = new Dictionary<ulong, string>();
string tdbPath = Path.Combine(path, "SerializedTypes.db");
if (!File.Exists(tdbPath))
string typesPath = Path.Combine(path, "SerializedTypes.db");
if (!File.Exists(typesPath))
{
return db;
}
using FileStream tdb = new FileStream(tdbPath, FileMode.Open, FileAccess.Read, FileShare.Read);
BinaryReader tdbReader = new BinaryReader(tdb);
using var mmf = MemoryMappedFile.CreateFromFile(typesPath, FileMode.Open);
using var accessor = mmf.CreateViewStream();
var version = tdbReader.ReadInt32();
var count = tdbReader.ReadInt32();
byte* ptr = null;
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
var dataReader = new UnmanagedDataReader(ptr, accessor.Length);
var version = dataReader.ReadInt();
var count = dataReader.ReadInt();
for (var i = 0; i < count; ++i)
{
var hash = tdbReader.ReadUInt64();
var typeName = tdbReader.ReadString();
var hash = dataReader.ReadULong();
var typeName = dataReader.ReadStringRaw();
db[hash] = typeName;
}
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
return db;
}
internal static void SerializeAll()
// Note: This is strictly on a background thread
internal static void PreSerializeAll(string path, ConcurrentQueue<Type> types)
{
foreach (var p in _registry)
{
p.Serialize();
p.Preserialize(path, types);
}
}
internal static void PostSerializeAll()
private static readonly HashSet<Type> _typesSet = [];
// Note: This is strictly on a background thread
internal static void WriteSnapshotAll(string path, ConcurrentQueue<Type> types)
{
foreach (var p in _registry)
{
p.PostSerialize();
}
}
internal static void PostDeserializeAll()
{
foreach (var p in _registry)
{
p.PostDeserialize();
}
}
public static void WriteSnapshot(string path, ConcurrentQueue<Type> types)
{
foreach (var entry in _registry)
{
entry.WriteSnapshot(path);
p.WriteSnapshot();
}
// Dedupe the queue.
@ -119,32 +113,62 @@ public abstract class Persistence
_typesSet.Clear();
}
private static HashSet<Type> _typesSet = new();
internal static void SerializeAll()
{
foreach (var p in _registry)
{
p.Serialize();
}
}
internal static void PostWorldSaveAll()
{
foreach (var p in _registry)
{
p.PostWorldSave();
}
}
internal static void PostDeserializeAll()
{
foreach (var p in _registry)
{
p.PostDeserialize();
}
}
public static void WriteSerializedTypesSnapshot(string path, HashSet<Type> types)
{
string tdbPath = Path.Combine(path, "SerializedTypes.db");
using var tdb = new BinaryFileWriter(tdbPath, false);
string typesPath = Path.Combine(path, "SerializedTypes.db");
using var fs = new FileStream(typesPath, FileMode.Create);
using var writer = new MemoryMapFileWriter(fs, 1024 * 1024 * 4);
tdb.Write(0); // version
tdb.Write(types.Count);
writer.Write(0); // version
writer.Write(types.Count);
foreach (var type in types)
{
var fullName = type.FullName;
tdb.Write(HashUtility.ComputeHash64(fullName));
tdb.Write(fullName);
writer.Write(HashUtility.ComputeHash64(fullName));
writer.WriteStringRaw(fullName);
}
}
// Serializes to memory buffers and run in parallel
public abstract void Serialize();
// Open file streams, MMFs, prepare data structures
// Note: This should only be run on a background thread
public abstract void Preserialize(string savePath, ConcurrentQueue<Type> types);
public abstract void WriteSnapshot(string savePath);
// Note: This should only be run on a background thread
public abstract void Serialize(IGenericSerializable e, int threadIndex);
// Note: This should only be run on a background thread
public abstract void WriteSnapshot();
public abstract void Serialize();
public abstract void Deserialize(string savePath, Dictionary<ulong, string> typesDb);
public virtual void PostSerialize()
public virtual void PostWorldSave()
{
}