fix: Adds ReadType/Write(Type) and improves type referencing (#1172)
## Changes * Improves type hashing by introducing xxHash3 (64bit) * Removes individual `tdb` files in favor of a single `SerializedTypes.db` file. This file is only used to identify a type that is being deserialized, which doesn't exist. * Adds duplicate type alias detection * Adds `AssemblyHandler.FindTypeByHash` View changed files whitespaces: https://github.com/modernuo/ModernUO/pull/1172/files?diff=split&w=1 ## SerializedTypes.db The serialized types file is used to get back the original name of a type in case it no longer exists in code. This can easily be necessary if a class is renamed in code and no `TypeAlias` is provided. ### Format byte[4] - version byte[4] - count --array-- byte[8] - xxHash byte[1] - flag, 0 - null, 1 - not null byte[n] - Full class name in UTF8 ### Example <img width="472" alt="SerializedTypes_Example" src="https://user-images.githubusercontent.com/3953314/195255429-31d24293-6bd1-419e-811b-07874dd0f78d.png"> ## Benchmarks Serialized 500 Type fields. The 8192bytes comes from the _ConcurrentQueue_ that would later be used for SerializedTypes. Note that the queue is never cleared, so it's size grew considerably. ```cs | Method | Mean | Error | StdDev | Allocated | |--------------------- |---------:|---------:|---------:|----------:| | BenchmarkXXHash | 18.44 us | 0.278 us | 0.260 us | 8192 B | | BenchmarkTypeStrings | 25.09 us | 0.292 us | 0.259 us | - | ``` TODO: * Add support in the Serialization Generator for `ReadType()` and `Write(Type)` * Remove `SetTypeRef` from Serialization Generator
This commit is contained in:
Kamron Batman
GitHub
parent
f268d5d4e2
commit
e1e30998ba
28 changed files with 616 additions and 291 deletions
|
|
@ -14,6 +14,7 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
|
|
@ -25,13 +26,11 @@ namespace Server;
|
|||
|
||||
public static class EntityPersistence
|
||||
{
|
||||
private const int _idxVersion = 1;
|
||||
|
||||
public static void WriteEntities<I, T>(
|
||||
IIndexInfo<I> indexInfo,
|
||||
Dictionary<I, T> entities,
|
||||
List<Type> types,
|
||||
string savePath,
|
||||
ConcurrentQueue<Type> types,
|
||||
out Dictionary<string, int> counts
|
||||
) where T : class, ISerializable
|
||||
{
|
||||
|
|
@ -44,20 +43,19 @@ public static class EntityPersistence
|
|||
PathUtility.EnsureDirectory(path);
|
||||
|
||||
string idxPath = Path.Combine(path, $"{typeName}.idx");
|
||||
string tdbPath = Path.Combine(path, $"{typeName}.tdb");
|
||||
string binPath = Path.Combine(path, $"{typeName}.bin");
|
||||
|
||||
using var idx = new BinaryFileWriter(idxPath, false);
|
||||
using var tdb = new BinaryFileWriter(tdbPath, false);
|
||||
using var bin = new BinaryFileWriter(binPath, true);
|
||||
using var idx = new BinaryFileWriter(idxPath, false, types);
|
||||
using var bin = new BinaryFileWriter(binPath, true, types);
|
||||
|
||||
idx.Write(1); // Version
|
||||
idx.Write(2); // Version
|
||||
idx.Write(entities.Count);
|
||||
foreach (var e in entities.Values)
|
||||
{
|
||||
long start = bin.Position;
|
||||
|
||||
idx.Write(e.TypeRef);
|
||||
var t = e.GetType();
|
||||
idx.Write(t);
|
||||
idx.Write(e.Serial);
|
||||
idx.Write(e.Created.Ticks);
|
||||
idx.Write(e.LastSerialized.Ticks);
|
||||
|
|
@ -73,12 +71,6 @@ public static class EntityPersistence
|
|||
counts[type] = (counts.TryGetValue(type, out var count) ? count : 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
tdb.Write(types.Count);
|
||||
for (int i = 0; i < types.Count; ++i)
|
||||
{
|
||||
tdb.Write(types[i].FullName);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
|
|
@ -90,6 +82,7 @@ public static class EntityPersistence
|
|||
public static Dictionary<I, T> LoadIndex<I, T>(
|
||||
string path,
|
||||
IIndexInfo<I> indexInfo,
|
||||
Dictionary<ulong, string> serializedTypes,
|
||||
out List<EntitySpan<T>> entities
|
||||
) where T : class, ISerializable
|
||||
{
|
||||
|
|
@ -99,11 +92,10 @@ public static class EntityPersistence
|
|||
var indexType = indexInfo.TypeName;
|
||||
|
||||
string indexPath = Path.Combine(path, indexType, $"{indexType}.idx");
|
||||
string typesPath = Path.Combine(path, indexType, $"{indexType}.tdb");
|
||||
|
||||
entities = new List<EntitySpan<T>>();
|
||||
|
||||
if (!File.Exists(indexPath) || !File.Exists(typesPath))
|
||||
if (!File.Exists(indexPath))
|
||||
{
|
||||
return map;
|
||||
}
|
||||
|
|
@ -111,58 +103,77 @@ public static class EntityPersistence
|
|||
using FileStream idx = new FileStream(indexPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
BinaryReader idxReader = new BinaryReader(idx);
|
||||
|
||||
using FileStream tdb = new FileStream(typesPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
BinaryReader tdbReader = new BinaryReader(tdb);
|
||||
|
||||
List<Tuple<ConstructorInfo, string>> types = ReadTypes<I>(tdbReader);
|
||||
|
||||
int count;
|
||||
var version = idxReader.ReadInt32();
|
||||
int count = idxReader.ReadInt32();
|
||||
|
||||
// Handle non-versioned (version 0).
|
||||
if (version > _idxVersion || idx.Length - 4 - version * 20 == 0)
|
||||
var ctorArguments = new[] { typeof(I) };
|
||||
List<ConstructorInfo> types;
|
||||
|
||||
string typesPath = Path.Combine(path, indexType, $"{indexType}.tdb");
|
||||
if (File.Exists(typesPath))
|
||||
{
|
||||
count = version;
|
||||
version = 0;
|
||||
using FileStream tdb = new FileStream(typesPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
BinaryReader tdbReader = new BinaryReader(tdb);
|
||||
types = ReadTypes(tdbReader, ctorArguments);
|
||||
tdbReader.Close();
|
||||
}
|
||||
else
|
||||
{
|
||||
count = idxReader.ReadInt32();
|
||||
types = null;
|
||||
}
|
||||
|
||||
// We must have a typeDb from SerializedTypes.db, or a tdb file
|
||||
if (serializedTypes == null && types == null)
|
||||
{
|
||||
return map;
|
||||
}
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
for (int i = 0; i < count; ++i)
|
||||
{
|
||||
var typeID = idxReader.ReadInt32();
|
||||
ConstructorInfo ctor;
|
||||
if (version >= 2)
|
||||
{
|
||||
var flag = idxReader.ReadByte();
|
||||
if (flag != 2)
|
||||
{
|
||||
throw new Exception($"Invalid type flag, expected 2 but received {flag}.");
|
||||
}
|
||||
|
||||
var hash = idxReader.ReadUInt64();
|
||||
serializedTypes!.TryGetValue(hash, out var typeName);
|
||||
ctor = GetConstructorFor(typeName, AssemblyHandler.FindTypeByHash(hash), ctorArguments);
|
||||
}
|
||||
else
|
||||
{
|
||||
ctor = types?[idxReader.ReadInt32()];
|
||||
}
|
||||
|
||||
var serial = idxReader.ReadUInt32();
|
||||
var created = version == 0 ? now : new DateTime(idxReader.ReadInt64(), DateTimeKind.Utc);
|
||||
var lastSerialized = version == 0 ? DateTime.MinValue : new DateTime(idxReader.ReadInt64(), DateTimeKind.Utc);
|
||||
var pos = idxReader.ReadInt64();
|
||||
var length = idxReader.ReadInt32();
|
||||
|
||||
Tuple<ConstructorInfo, string> objs = types[typeID];
|
||||
|
||||
if (objs == null)
|
||||
if (ctor == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
ConstructorInfo ctor = objs.Item1;
|
||||
I indexer = indexInfo.CreateIndex(serial);
|
||||
|
||||
ctorArgs[0] = indexer;
|
||||
|
||||
if (ctor.Invoke(ctorArgs) is T t)
|
||||
if (ctor.Invoke(ctorArgs) is T entity)
|
||||
{
|
||||
t.Created = created;
|
||||
t.LastSerialized = lastSerialized;
|
||||
entities.Add(new EntitySpan<T>(t, typeID, pos, length));
|
||||
map[indexer] = t;
|
||||
entity.Created = created;
|
||||
entity.LastSerialized = lastSerialized;
|
||||
entities.Add(new EntitySpan<T>(entity, pos, length));
|
||||
map[indexer] = entity;
|
||||
}
|
||||
}
|
||||
|
||||
tdbReader.Close();
|
||||
idxReader.Close();
|
||||
|
||||
return map;
|
||||
|
|
@ -171,6 +182,7 @@ public static class EntityPersistence
|
|||
public static void LoadData<I, T>(
|
||||
string path,
|
||||
IIndexInfo<I> indexInfo,
|
||||
Dictionary<ulong, string> serializedTypes,
|
||||
List<EntitySpan<T>> entities
|
||||
) where T : class, ISerializable
|
||||
{
|
||||
|
|
@ -211,7 +223,7 @@ public static class EntityPersistence
|
|||
var buffer = GC.AllocateUninitializedArray<byte>(entry.Length);
|
||||
if (br == null)
|
||||
{
|
||||
br = new BufferReader(buffer, t.LastSerialized);
|
||||
br = new BufferReader(buffer, t.LastSerialized, serializedTypes);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -236,7 +248,7 @@ public static class EntityPersistence
|
|||
|
||||
if (error == null)
|
||||
{
|
||||
t.InitializeSaveBuffer(buffer);
|
||||
t.InitializeSaveBuffer(buffer, World.SerializedTypes);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -265,50 +277,52 @@ public static class EntityPersistence
|
|||
}
|
||||
}
|
||||
|
||||
private static List<Tuple<ConstructorInfo, string>> ReadTypes<I>(BinaryReader tdbReader)
|
||||
private static ConstructorInfo GetConstructorFor(string typeName, Type t, Type[] constructorTypes)
|
||||
{
|
||||
var constructorTypes = new[] { typeof(I) };
|
||||
if (t?.IsAbstract != false)
|
||||
{
|
||||
Console.WriteLine("failed");
|
||||
|
||||
var issue = t?.IsAbstract == true ? "marked abstract" : "not found";
|
||||
|
||||
Console.WriteLine($"Error: Type '{typeName}' was {issue}. Delete all of those types? (y/n)");
|
||||
|
||||
if (Console.ReadKey(true).Key == ConsoleKey.Y)
|
||||
{
|
||||
Console.WriteLine("Loading...");
|
||||
return null;
|
||||
}
|
||||
|
||||
Console.WriteLine("Types will not be deleted. An exception will be thrown.");
|
||||
|
||||
throw new Exception($"Bad type '{typeName}'");
|
||||
}
|
||||
|
||||
var ctor = t.GetConstructor(constructorTypes);
|
||||
|
||||
if (ctor == null)
|
||||
{
|
||||
throw new Exception($"Type '{t}' does not have a serialization constructor");
|
||||
}
|
||||
|
||||
return ctor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy ReadTypes for backward compatibility with old saves that still have a tdb file
|
||||
*/
|
||||
private static List<ConstructorInfo> ReadTypes(BinaryReader tdbReader, Type[] ctorArguments)
|
||||
{
|
||||
var count = tdbReader.ReadInt32();
|
||||
|
||||
var types = new List<Tuple<ConstructorInfo, string>>(count);
|
||||
var types = new List<ConstructorInfo>(count);
|
||||
|
||||
for (var i = 0; i < count; ++i)
|
||||
{
|
||||
var typeName = tdbReader.ReadString();
|
||||
|
||||
var t = AssemblyHandler.FindTypeByFullName(typeName, false);
|
||||
|
||||
if (t?.IsAbstract != false)
|
||||
{
|
||||
Console.WriteLine("failed");
|
||||
|
||||
var issue = t?.IsAbstract == true ? "marked abstract" : "not found";
|
||||
|
||||
Console.WriteLine($"Error: Type '{typeName}' was {issue}. Delete all of those types? (y/n)");
|
||||
|
||||
if (Console.ReadKey(true).Key == ConsoleKey.Y)
|
||||
{
|
||||
types.Add(null);
|
||||
Console.WriteLine("Loading...");
|
||||
continue;
|
||||
}
|
||||
|
||||
Console.WriteLine("Types will not be deleted. An exception will be thrown.");
|
||||
|
||||
throw new Exception($"Bad type '{typeName}'");
|
||||
}
|
||||
|
||||
var ctor = t.GetConstructor(constructorTypes);
|
||||
|
||||
if (ctor != null)
|
||||
{
|
||||
types.Add(new Tuple<ConstructorInfo, string>(ctor, typeName));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception($"Type '{t}' does not have a serialization constructor");
|
||||
}
|
||||
var ctor = GetConstructorFor(typeName, t, ctorArguments);
|
||||
types.Add(ctor);
|
||||
}
|
||||
|
||||
return types;
|
||||
|
|
|
|||
|
|
@ -19,16 +19,13 @@ public struct EntitySpan<T> where T : ISerializable
|
|||
{
|
||||
public T Entity { get; }
|
||||
|
||||
public int TypeID { get; }
|
||||
|
||||
public long Position { get; }
|
||||
|
||||
public int Length { get; }
|
||||
|
||||
public EntitySpan(T entity, int typeID, long position, int length)
|
||||
public EntitySpan(T entity, long position, int length)
|
||||
{
|
||||
Entity = entity;
|
||||
TypeID = typeID;
|
||||
Position = position;
|
||||
Length = length;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,12 +38,12 @@ public enum WorldState
|
|||
|
||||
public static class World
|
||||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(World));
|
||||
private static ILogger logger = LogFactory.GetLogger(typeof(World));
|
||||
|
||||
private static readonly ManualResetEvent m_DiskWriteHandle = new(true);
|
||||
private static readonly Dictionary<Serial, IEntity> _pendingAdd = new();
|
||||
private static readonly Dictionary<Serial, IEntity> _pendingDelete = new();
|
||||
private static readonly ConcurrentQueue<Item> _decayQueue = new();
|
||||
private static ManualResetEvent m_DiskWriteHandle = new(true);
|
||||
private static Dictionary<Serial, IEntity> _pendingAdd = new();
|
||||
private static Dictionary<Serial, IEntity> _pendingDelete = new();
|
||||
private static ConcurrentQueue<Item> _decayQueue = new();
|
||||
|
||||
private static string _tempSavePath; // Path to the temporary folder for the save
|
||||
private static bool _enableSaveStats;
|
||||
|
|
@ -125,10 +125,6 @@ public static class World
|
|||
|
||||
private static void OutOfMemory(string message) => throw new OutOfMemoryException(message);
|
||||
|
||||
internal static List<Type> ItemTypes { get; } = new();
|
||||
internal static List<Type> MobileTypes { get; } = new();
|
||||
internal static List<Type> GuildTypes { get; } = new();
|
||||
|
||||
public static string SavePath { get; private set; }
|
||||
|
||||
public static WorldState WorldState { get; private set; }
|
||||
|
|
@ -232,15 +228,15 @@ public static class World
|
|||
public static void Broadcast(int hue, bool ascii, string format, params object[] args) =>
|
||||
Broadcast(hue, ascii, string.Format(format, args));
|
||||
|
||||
internal static void LoadEntities(string basePath)
|
||||
internal static void LoadEntities(string basePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
IIndexInfo<Serial> itemIndexInfo = new EntityTypeIndex("Items");
|
||||
IIndexInfo<Serial> mobileIndexInfo = new EntityTypeIndex("Mobiles");
|
||||
IIndexInfo<Serial> guildIndexInfo = new EntityTypeIndex("Guilds");
|
||||
|
||||
Mobiles = EntityPersistence.LoadIndex(basePath, mobileIndexInfo, out List<EntitySpan<Mobile>> mobiles);
|
||||
Items = EntityPersistence.LoadIndex(basePath, itemIndexInfo, out List<EntitySpan<Item>> items);
|
||||
Guilds = EntityPersistence.LoadIndex(basePath, guildIndexInfo, out List<EntitySpan<BaseGuild>> guilds);
|
||||
Mobiles = EntityPersistence.LoadIndex(basePath, mobileIndexInfo, typesDb, out List<EntitySpan<Mobile>> mobiles);
|
||||
Items = EntityPersistence.LoadIndex(basePath, itemIndexInfo, typesDb, out List<EntitySpan<Item>> items);
|
||||
Guilds = EntityPersistence.LoadIndex(basePath, guildIndexInfo, typesDb, out List<EntitySpan<BaseGuild>> guilds);
|
||||
|
||||
if (Mobiles.Count > 0)
|
||||
{
|
||||
|
|
@ -257,9 +253,9 @@ public static class World
|
|||
_lastGuild = Guilds.Keys.Max();
|
||||
}
|
||||
|
||||
EntityPersistence.LoadData(basePath, mobileIndexInfo, mobiles);
|
||||
EntityPersistence.LoadData(basePath, itemIndexInfo, items);
|
||||
EntityPersistence.LoadData(basePath, guildIndexInfo, guilds);
|
||||
EntityPersistence.LoadData(basePath, mobileIndexInfo, typesDb, mobiles);
|
||||
EntityPersistence.LoadData(basePath, itemIndexInfo, typesDb, items);
|
||||
EntityPersistence.LoadData(basePath, guildIndexInfo, typesDb, guilds);
|
||||
}
|
||||
|
||||
public static void Load()
|
||||
|
|
@ -392,9 +388,9 @@ public static class World
|
|||
IIndexInfo<Serial> mobileIndexInfo = new EntityTypeIndex("Mobiles");
|
||||
IIndexInfo<Serial> guildIndexInfo = new EntityTypeIndex("Guilds");
|
||||
|
||||
EntityPersistence.WriteEntities(mobileIndexInfo, Mobiles, MobileTypes, basePath, out var mobileCounts);
|
||||
EntityPersistence.WriteEntities(itemIndexInfo, Items, ItemTypes, basePath, out var itemCounts);
|
||||
EntityPersistence.WriteEntities(guildIndexInfo, Guilds, GuildTypes, basePath, out var guildCounts);
|
||||
EntityPersistence.WriteEntities(mobileIndexInfo, Mobiles, basePath, SerializedTypes, out var mobileCounts);
|
||||
EntityPersistence.WriteEntities(itemIndexInfo, Items, basePath, SerializedTypes, out var itemCounts);
|
||||
EntityPersistence.WriteEntities(guildIndexInfo, Guilds, basePath, SerializedTypes, out var guildCounts);
|
||||
|
||||
if (_enableSaveStats)
|
||||
{
|
||||
|
|
@ -413,7 +409,7 @@ public static class World
|
|||
var watch = Stopwatch.StartNew();
|
||||
logger.Information("Writing world save snapshot");
|
||||
|
||||
Persistence.WriteSnapshot(tempPath);
|
||||
Persistence.WriteSnapshot(tempPath, SerializedTypes);
|
||||
|
||||
watch.Stop();
|
||||
|
||||
|
|
@ -444,6 +440,9 @@ public static class World
|
|||
}
|
||||
}
|
||||
|
||||
// Clear types
|
||||
SerializedTypes.Clear();
|
||||
|
||||
m_DiskWriteHandle.Set();
|
||||
|
||||
Core.LoopContext.Post(FinishWorldSave);
|
||||
|
|
@ -463,6 +462,39 @@ public static class World
|
|||
|
||||
private static DateTime _serializationStart;
|
||||
|
||||
/**
|
||||
* Duplicates can be weeded out asynchronously while flushing
|
||||
* If performance becomes a problem, we need to build a dual mode concurrent array.
|
||||
*
|
||||
****************************************************** Proposal ******************************************************
|
||||
* The structure is initialized with a large capacity to avoid unnecessary resizing.
|
||||
* Write Mode:
|
||||
* - Multiple threads can add a single, or a range of elements concurrently.
|
||||
* - Elements can be Peeked, but there are no guarantees.
|
||||
* - To resize the internal array, replaced it with the next size up from an array pool.
|
||||
* - The structure cannot be cleared in this mode.
|
||||
*
|
||||
* Read Mode:
|
||||
* - The array can be read from multiple threads using a ref struct enumerator.
|
||||
* - Elements cannot be added or reassigned.
|
||||
* - Cleared by replacing the internal array with another one from the pool.
|
||||
* - Note: Upon clearing, the existing array is not sent back to the pool until there are zero enumerators.
|
||||
*
|
||||
* Enumeration:
|
||||
* - Multiple threads can enumerate while in read mode. The enumerator will Interlocked.Increment a read counter.
|
||||
* - Upon dispose of the enumerator, the read counter will be lowered with an Interlocked.Decrement
|
||||
* - When the read counter reaches 0, if there is a cleared array, the array is sent back to the pool zeroed.
|
||||
*
|
||||
* Notes:
|
||||
* - Elements can never be removed.
|
||||
*
|
||||
* How is this different from ConcurrentQueue?
|
||||
* The functionality is very similar, except the constraints allow the implementation to be done without locks.
|
||||
* Since this implementation uses pooled arrays, allocations will approach zero over time.
|
||||
**********************************************************************************************************************
|
||||
*/
|
||||
public static ConcurrentQueue<Type> SerializedTypes { get; } = new();
|
||||
|
||||
internal static void SaveEntities()
|
||||
{
|
||||
_serializationStart = DateTime.UtcNow;
|
||||
|
|
@ -479,7 +511,7 @@ public static class World
|
|||
EnqueueForDecay(item);
|
||||
}
|
||||
|
||||
entity.Serialize();
|
||||
entity.Serialize(SerializedTypes);
|
||||
}
|
||||
|
||||
public static void Save()
|
||||
|
|
@ -632,7 +664,10 @@ public static class World
|
|||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void AddGuild(BaseGuild guild) => Guilds[guild.Serial] = guild;
|
||||
public static void AddGuild(BaseGuild guild)
|
||||
{
|
||||
Guilds[guild.Serial] = guild;
|
||||
}
|
||||
|
||||
public static void RemoveEntity<T>(T entity) where T : class, IEntity
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue