fix: Adds ReadType/Write(Type) and improves type referencing (#1172)

## Changes
* Improves type hashing by introducing xxHash3 (64bit)
* Removes individual `tdb` files in favor of a single `SerializedTypes.db` file. This file is only used to identify a type that is being deserialized, which doesn't exist.
* Adds duplicate type alias detection
* Adds `AssemblyHandler.FindTypeByHash`

View changed files whitespaces: https://github.com/modernuo/ModernUO/pull/1172/files?diff=split&w=1

## SerializedTypes.db
The serialized types file is used to get back the original name of a type in case it no longer exists in code. This can easily be necessary if a class is renamed in code and no `TypeAlias` is provided.

### Format
byte[4] - version
byte[4] - count
--array--
byte[8] - xxHash
byte[1] - flag, 0 - null, 1 - not null
byte[n] - Full class name in UTF8

### Example
<img width="472" alt="SerializedTypes_Example" src="https://user-images.githubusercontent.com/3953314/195255429-31d24293-6bd1-419e-811b-07874dd0f78d.png">

## Benchmarks
Serialized 500 Type fields. The 8192bytes comes from the _ConcurrentQueue_ that would later be used for SerializedTypes.
Note that the queue is never cleared, so it's size grew considerably.
```cs
|               Method |     Mean |    Error |   StdDev | Allocated |
|--------------------- |---------:|---------:|---------:|----------:|
|      BenchmarkXXHash | 18.44 us | 0.278 us | 0.260 us |    8192 B |
| BenchmarkTypeStrings | 25.09 us | 0.292 us | 0.259 us |         - |
```

TODO:
* Add support in the Serialization Generator for `ReadType()` and `Write(Type)`
* Remove `SetTypeRef` from Serialization Generator
This commit is contained in:
Kamron Batman 2022-10-11 22:17:22 -07:00 committed by GitHub
parent f268d5d4e2
commit e1e30998ba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 616 additions and 291 deletions

View file

@ -14,13 +14,14 @@
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
namespace Server;
public class Persistence
public static class Persistence
{
public const int DefaultPriority = 100;
@ -30,7 +31,7 @@ public class Persistence
string name,
Action serializer,
Action<string> snapshotWriter,
Action<string> deserializer,
Action<string, Dictionary<ulong, string>> deserializer,
int priority = DefaultPriority
)
{
@ -50,24 +51,79 @@ public class Persistence
public static void Load(string path)
{
var typesDb = LoadTypes(path);
// This should probably not be parallel since Mobiles must be loaded before Items
foreach (var entry in _registry)
{
entry.Deserialize(path);
entry.Deserialize(path, typesDb);
}
}
private static Dictionary<ulong, string> LoadTypes(string path)
{
var db = new Dictionary<ulong, string>();
string tdbPath = Path.Combine(path, "SerializedTypes.db");
if (!File.Exists(tdbPath))
{
return db;
}
using FileStream tdb = new FileStream(tdbPath, FileMode.Open, FileAccess.Read, FileShare.Read);
BinaryReader tdbReader = new BinaryReader(tdb);
var version = tdbReader.ReadInt32();
var count = tdbReader.ReadInt32();
for (var i = 0; i < count; ++i)
{
var hash = tdbReader.ReadUInt64();
var typeName = tdbReader.ReadString();
db[hash] = typeName;
}
return db;
}
public static void Serialize()
{
Parallel.ForEach(_registry, entry => entry.Serialize());
}
public static void WriteSnapshot(string path)
public static void WriteSnapshot(string path, ConcurrentQueue<Type> types)
{
foreach (var entry in _registry)
{
entry.WriteSnapshot(path);
}
// Dedupe the queue.
foreach (var type in types)
{
_typesSet.Add(type);
}
WriteSerializedTypesSnapshot(path, _typesSet);
_typesSet.Clear();
}
private static HashSet<Type> _typesSet = new();
public static void WriteSerializedTypesSnapshot(string path, HashSet<Type> types)
{
string tdbPath = Path.Combine(path, "SerializedTypes.db");
using var tdb = new BinaryFileWriter(tdbPath, false);
tdb.Write(0); // version
tdb.Write(types.Count);
foreach (var type in types)
{
var fullName = type.FullName;
tdb.Write(HashUtility.ComputeHash64(fullName));
tdb.Write(fullName);
}
}
public record RegistryEntry
@ -76,7 +132,7 @@ public class Persistence
public int Priority { get; init; }
public Action Serialize { get; init; } // Serializing to memory buffers
public Action<string> WriteSnapshot { get; init; }
public Action<string> Deserialize { get; init; }
public Action<string, Dictionary<ulong, string>> Deserialize { get; init; }
}
internal class RegistryEntryComparer : IComparer<RegistryEntry>