fix: Adds ReadType/Write(Type) and improves type referencing (#1172)

## Changes
* Improves type hashing by introducing xxHash3 (64bit)
* Removes individual `tdb` files in favor of a single `SerializedTypes.db` file. This file is only used to identify a type that is being deserialized, which doesn't exist.
* Adds duplicate type alias detection
* Adds `AssemblyHandler.FindTypeByHash`

View changed files whitespaces: https://github.com/modernuo/ModernUO/pull/1172/files?diff=split&w=1

## SerializedTypes.db
The serialized types file is used to get back the original name of a type in case it no longer exists in code. This can easily be necessary if a class is renamed in code and no `TypeAlias` is provided.

### Format
byte[4] - version
byte[4] - count
--array--
byte[8] - xxHash
byte[1] - flag, 0 - null, 1 - not null
byte[n] - Full class name in UTF8

### Example
<img width="472" alt="SerializedTypes_Example" src="https://user-images.githubusercontent.com/3953314/195255429-31d24293-6bd1-419e-811b-07874dd0f78d.png">

## Benchmarks
Serialized 500 Type fields. The 8192bytes comes from the _ConcurrentQueue_ that would later be used for SerializedTypes.
Note that the queue is never cleared, so it's size grew considerably.
```cs
|               Method |     Mean |    Error |   StdDev | Allocated |
|--------------------- |---------:|---------:|---------:|----------:|
|      BenchmarkXXHash | 18.44 us | 0.278 us | 0.260 us |    8192 B |
| BenchmarkTypeStrings | 25.09 us | 0.292 us | 0.259 us |         - |
```

TODO:
* Add support in the Serialization Generator for `ReadType()` and `Write(Type)`
* Remove `SetTypeRef` from Serialization Generator
This commit is contained in:
Kamron Batman 2022-10-11 22:17:22 -07:00 committed by GitHub
parent f268d5d4e2
commit e1e30998ba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 616 additions and 291 deletions

View file

@ -14,6 +14,8 @@
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.MemoryMappedFiles;
using System.Threading.Tasks;
@ -27,30 +29,53 @@ public static class AdhocPersistence
* Note: The buffer may not be the same after returning from the function if more data is written
* than the initial buffer can handle.
*/
public static BufferWriter Serialize(Action<IGenericWriter> serializer)
public static BufferWriter Serialize(Action<IGenericWriter> serializer, ConcurrentQueue<Type> types)
{
var saveBuffer = new BufferWriter(true);
var saveBuffer = new BufferWriter(true, types);
serializer(saveBuffer);
return saveBuffer;
}
/**
* Writes a buffer to disk. This function should be called asynchronously.
* Writes the filePath for the binary data, and an accompanying SerializedTypes.db file of all possible types.
*/
public static void WriteSnapshot(string filePath, Span<byte> buffer)
public static void WriteSnapshot(FileInfo file, Span<byte> buffer)
{
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
var file = new FileInfo(fullPath);
PathUtility.EnsureDirectory(file.DirectoryName);
var dirPath = file.DirectoryName;
PathUtility.EnsureDirectory(dirPath);
using var fs = new FileStream(fullPath, FileMode.Create, FileAccess.Write);
using var fs = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
fs.Write(buffer);
}
public static void SerializeAndSnapshot(string filePath, Action<IGenericWriter> serializer)
/**
* Serializes to a memory buffer synchronously, then flushes to the path asynchronously.
* See WriteSnapshot for more info about how the snapshot.
*/
public static void SerializeAndSnapshot(string filePath, Action<IGenericWriter> serializer, ConcurrentQueue<Type> types = null)
{
var saveBuffer = Serialize(serializer);
Task.Run(() => { WriteSnapshot(filePath, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position)); });
types ??= new ConcurrentQueue<Type>();
var saveBuffer = Serialize(serializer, types);
Task.Run(
() =>
{
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
var file = new FileInfo(fullPath);
WriteSnapshot(file, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position));
// TODO: Create a PooledHashSet if performance becomes an issue.
var typesSet = new HashSet<Type>();
// Dedupe the queue.
foreach (var type in types)
{
typesSet.Add(type);
}
Persistence.WriteSerializedTypesSnapshot(file.DirectoryName, typesSet);
});
}
public static void Deserialize(string filePath, Action<IGenericReader> deserializer)

View file

@ -14,27 +14,34 @@
*************************************************************************/
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using Server.Buffers;
using Server.Collections;
using Server.Logging;
using Server.Text;
namespace Server;
public class BinaryFileReader : IGenericReader, IDisposable
{
private static readonly ILogger logger = LogFactory.GetLogger(typeof(BinaryFileReader));
private Dictionary<ulong, string> _typesDb;
private BinaryReader _reader;
private Encoding _encoding;
public BinaryFileReader(BinaryReader br, Encoding encoding = null)
public BinaryFileReader(BinaryReader br, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
{
_reader = br;
_encoding = encoding ?? TextEncoding.UTF8;
_typesDb = typesDb;
}
public BinaryFileReader(Stream stream, Encoding encoding = null) : this(new BinaryReader(stream), encoding)
public BinaryFileReader(Stream stream, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
: this(new BinaryReader(stream), typesDb, encoding)
{
}
@ -46,23 +53,20 @@ public class BinaryFileReader : IGenericReader, IDisposable
public DateTime LastSerialized { get; init; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public string ReadString(bool intern = false)
{
if (!ReadBool())
{
return null;
}
public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null;
public string ReadStringRaw(bool intern = false)
{
var length = ((IGenericReader)this).ReadEncodedInt();
if (length <= 0)
{
return intern ? Utility.Intern("") : "";
return "".Intern();
}
byte[] buffer = STArrayPool<byte>.Shared.Rent(length);
var str = TextEncoding.GetString(buffer.AsSpan(0, length), _encoding);
STArrayPool<byte>.Shared.Return(buffer);
return intern ? Utility.Intern(str) : str;
return intern ? str.Intern() : str;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@ -101,6 +105,62 @@ public class BinaryFileReader : IGenericReader, IDisposable
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Serial ReadSerial() => (Serial)_reader.ReadUInt32();
public Type ReadType() =>
ReadByte() switch
{
0 => null,
1 => AssemblyHandler.FindTypeByFullName(ReadStringRaw()), // Backward compatibility
2 => ReadTypeByHash()
};
public Type ReadTypeByHash()
{
var hash = ReadULong();
var t = AssemblyHandler.FindTypeByHash(hash);
if (t != null)
{
return t;
}
if (_typesDb == null)
{
logger.Error(
new Exception($"The file SerializedTypes.db was not loaded. Type hash '{hash}' could not be found."),
"Invalid {Hash} at position {Position}",
hash,
Position
);
return null;
}
if (!_typesDb.TryGetValue(hash, out var typeName))
{
logger.Error(
new Exception($"Type hash '{hash}' is not present in the serialized types database."),
"Invalid type hash {Hash} at position {Position}",
hash,
Position
);
return null;
}
t = AssemblyHandler.FindTypeByFullName(typeName, false);
if (t == null)
{
logger.Error(
new Exception($"Type '{typeName}' was not found."),
"Type {Type} was not found.",
typeName
);
}
return t;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int Read(Span<byte> buffer) => _reader.Read(buffer);

View file

@ -14,6 +14,7 @@
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.IO;
namespace Server;
@ -23,11 +24,11 @@ public class BinaryFileWriter : BufferWriter, IDisposable
private readonly Stream _file;
private long _position;
public BinaryFileWriter(string filename, bool prefixStr) :
this(new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None), prefixStr)
public BinaryFileWriter(string filename, bool prefixStr, ConcurrentQueue<Type> types = null) :
this(new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None), prefixStr, types)
{}
public BinaryFileWriter(Stream stream, bool prefixStr) : base(prefixStr)
public BinaryFileWriter(Stream stream, bool prefixStr, ConcurrentQueue<Type> types = null) : base(prefixStr, types)
{
_file = stream;
_position = _file.Position;

View file

@ -15,30 +15,40 @@
using System;
using System.Buffers.Binary;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using Server.Collections;
using Server.Logging;
using Server.Text;
namespace Server;
public class BufferReader : IGenericReader
{
private static readonly ILogger logger = LogFactory.GetLogger(typeof(BufferReader));
private Dictionary<ulong, string> _typesDb;
private Encoding _encoding;
private byte[] _buffer;
private int _position;
public long Position => _position;
public BufferReader(byte[] buffer, Encoding encoding = null)
public BufferReader(byte[] buffer, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
{
_buffer = buffer;
_encoding = encoding ?? TextEncoding.UTF8;
_typesDb = typesDb;
}
public BufferReader(byte[] buffer, DateTime lastSerialized) : this(buffer) => LastSerialized = lastSerialized;
public BufferReader(byte[] buffer, DateTime lastSerialized, Dictionary<ulong, string> typesDb = null) : this(buffer)
{
LastSerialized = lastSerialized;
_typesDb = typesDb;
}
public void Reset(byte[] newBuffer, out byte[] oldBuffer)
{
@ -49,22 +59,20 @@ public class BufferReader : IGenericReader
public DateTime LastSerialized { get; init; }
public string ReadString(bool intern = false)
{
if (!ReadBool())
{
return null;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null;
public string ReadStringRaw(bool intern = false)
{
var length = ((IGenericReader)this).ReadEncodedInt();
if (length <= 0)
{
return intern ? Utility.Intern("") : "";
return "".Intern();
}
var str = TextEncoding.GetString(_buffer.AsSpan(_position, length), _encoding);
_position += length;
return intern ? Utility.Intern(str) : str;
return intern ? str.Intern() : str;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@ -143,6 +151,62 @@ public class BufferReader : IGenericReader
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Serial ReadSerial() => (Serial)ReadUInt();
public Type ReadType() =>
ReadByte() switch
{
0 => null,
1 => AssemblyHandler.FindTypeByFullName(ReadStringRaw()), // Backward compatibility
2 => ReadTypeByHash()
};
public Type ReadTypeByHash()
{
var hash = ReadULong();
var t = AssemblyHandler.FindTypeByHash(hash);
if (t != null)
{
return t;
}
if (_typesDb == null)
{
logger.Error(
new Exception($"The file SerializedTypes.db was not loaded. Type hash '{hash}' could not be found."),
"Invalid {Hash} at position {Position}",
hash,
Position
);
return null;
}
if (!_typesDb.TryGetValue(hash, out var typeName))
{
logger.Error(
new Exception($"Type hash '{hash}' is not present in the serialized types database."),
"Invalid type hash {Hash} at position {Position}",
hash,
Position
);
return null;
}
t = AssemblyHandler.FindTypeByFullName(typeName, false);
if (t == null)
{
logger.Error(
new Exception($"Type '{typeName}' was not found."),
"Type {Type} was not found.",
typeName
);
}
return t;
}
public int Read(Span<byte> buffer)
{
var length = buffer.Length;

View file

@ -2,7 +2,7 @@
* ModernUO *
* Copyright 2019-2022 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: BufferedFileWriter.cs *
* File: BufferWriter.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
@ -14,6 +14,7 @@
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
@ -25,8 +26,9 @@ namespace Server;
public class BufferWriter : IGenericWriter
{
private readonly Encoding m_Encoding;
private readonly bool m_PrefixStrings;
private ConcurrentQueue<Type> _types;
private Encoding _encoding;
private bool _prefixStrings;
private long _bytesWritten;
private long _index;
@ -53,22 +55,24 @@ public class BufferWriter : IGenericWriter
private byte[] _buffer;
public BufferWriter(byte[] buffer, bool prefixStr)
public BufferWriter(byte[] buffer, bool prefixStr, ConcurrentQueue<Type> types = null)
{
m_PrefixStrings = prefixStr;
m_Encoding = TextEncoding.UTF8;
_prefixStrings = prefixStr;
_encoding = TextEncoding.UTF8;
_buffer = buffer;
_types = types;
}
public BufferWriter(bool prefixStr) : this(0, prefixStr)
public BufferWriter(bool prefixStr, ConcurrentQueue<Type> types = null) : this(0, prefixStr, types)
{
}
public BufferWriter(int count, bool prefixStr)
public BufferWriter(int count, bool prefixStr, ConcurrentQueue<Type> types = null)
{
m_PrefixStrings = prefixStr;
m_Encoding = TextEncoding.UTF8;
_prefixStrings = prefixStr;
_encoding = TextEncoding.UTF8;
_buffer = GC.AllocateUninitializedArray<byte>(count < 1 ? BufferSize : count);
_types = types;
}
public virtual long Position => Index;
@ -135,6 +139,7 @@ public class BufferWriter : IGenericWriter
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(BitArray bitArray)
{
var byteLength = BitArray.GetByteArrayLengthFromBitLength(bitArray.Length);
@ -145,6 +150,7 @@ public class BufferWriter : IGenericWriter
Index += byteLength;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public virtual long Seek(long offset, SeekOrigin origin)
{
Debug.Assert(
@ -168,9 +174,10 @@ public class BufferWriter : IGenericWriter
});
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(string value)
{
if (m_PrefixStrings)
if (_prefixStrings)
{
if (value == null)
{
@ -188,6 +195,7 @@ public class BufferWriter : IGenericWriter
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(long value)
{
FlushIfNeeded(8);
@ -202,6 +210,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 56);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(ulong value)
{
FlushIfNeeded(8);
@ -216,6 +225,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 56);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(int value)
{
FlushIfNeeded(4);
@ -226,6 +236,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 24);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(uint value)
{
FlushIfNeeded(4);
@ -236,6 +247,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 24);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(short value)
{
FlushIfNeeded(2);
@ -244,6 +256,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 8);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(ushort value)
{
FlushIfNeeded(2);
@ -252,6 +265,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 8);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe void Write(double value)
{
FlushIfNeeded(8);
@ -264,6 +278,7 @@ public class BufferWriter : IGenericWriter
Index += 8;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe void Write(float value)
{
FlushIfNeeded(4);
@ -300,9 +315,25 @@ public class BufferWriter : IGenericWriter
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(Serial serial) => Write(serial.Value);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(Type type)
{
if (type == null)
{
Write((byte)0);
}
else
{
Write((byte)0x2); // xxHash3 64bit
Write(AssemblyHandler.GetTypeHash(type));
_types?.Enqueue(type);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal void InternalWriteString(string value)
{
var remaining = m_Encoding.GetByteCount(value);
var remaining = _encoding.GetByteCount(value);
((IGenericWriter)this).WriteEncodedInt(remaining);
@ -313,14 +344,14 @@ public class BufferWriter : IGenericWriter
// It is much faster to encode to stack buffer, then copy to the real buffer
Span<byte> span = stackalloc byte[Math.Min(BufferSize, 256)];
var maxChars = span.Length / m_Encoding.GetMaxByteCount(1);
var maxChars = span.Length / _encoding.GetMaxByteCount(1);
var charsLeft = value.Length;
var current = 0;
while (charsLeft > 0)
{
var charCount = Math.Min(charsLeft, maxChars);
var bytesWritten = m_Encoding.GetBytes(value.AsSpan(current, charCount), span);
var bytesWritten = _encoding.GetBytes(value.AsSpan(current, charCount), span);
remaining -= bytesWritten;
charsLeft -= charCount;
current += charCount;

View file

@ -14,6 +14,8 @@
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
namespace Server;
@ -31,22 +33,22 @@ public static class GenericPersistence
void Serialize()
{
saveBuffer ??= new BufferWriter(true);
saveBuffer ??= new BufferWriter(true, World.SerializedTypes);
saveBuffer.Seek(0, SeekOrigin.Begin);
serializer(saveBuffer);
}
void WriterSnapshot(string savePath)
void WriteSnapshot(string savePath)
{
string binPath = Path.Combine(savePath, name, $"{name}.bin");
var buffer = saveBuffer!.Buffer.AsSpan(0, (int)saveBuffer.Position);
AdhocPersistence.WriteSnapshot(binPath, buffer);
AdhocPersistence.WriteSnapshot(new FileInfo(binPath), buffer);
}
void Deserialize(string savePath) =>
void Deserialize(string savePath, Dictionary<ulong, string> typesDb) =>
AdhocPersistence.Deserialize(Path.Combine(savePath, name, $"{name}.bin"), deserializer);
Persistence.Register(name, Serialize, WriterSnapshot, Deserialize, priority);
Persistence.Register(name, Serialize, WriteSnapshot, Deserialize, priority);
}
}

View file

@ -38,6 +38,7 @@ public interface IGenericReader
sbyte ReadSByte();
bool ReadBool();
Serial ReadSerial();
Type ReadType();
DateTime ReadDateTime() => new(ReadLong(), DateTimeKind.Utc);
TimeSpan ReadTimeSpan() => new(ReadLong());

View file

@ -37,6 +37,7 @@ public interface IGenericWriter
void Write(sbyte value);
void Write(bool value);
void Write(Serial serial);
void Write(Type type);
void Write(DateTime value)
{

View file

@ -14,6 +14,7 @@
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.IO;
namespace Server;
@ -28,7 +29,6 @@ public interface ISerializable
long SavePosition { get; protected internal set; }
BufferWriter SaveBuffer { get; protected internal set; }
int TypeRef { get; }
Serial Serial { get; }
// Executed on every entity, before it's serialized.
@ -39,11 +39,9 @@ public interface ISerializable
void Delete();
bool Deleted { get; }
void SetTypeRef(Type type);
public void InitializeSaveBuffer(byte[] buffer)
public void InitializeSaveBuffer(byte[] buffer, ConcurrentQueue<Type> types)
{
SaveBuffer = new BufferWriter(buffer, true);
SaveBuffer = new BufferWriter(buffer, true, types);
if (World.DirtyTrackingEnabled)
{
SavePosition = SaveBuffer.Position;
@ -54,9 +52,9 @@ public interface ISerializable
}
}
public void Serialize()
public void Serialize(ConcurrentQueue<Type> types)
{
SaveBuffer ??= new BufferWriter(true);
SaveBuffer ??= new BufferWriter(true, types);
BeforeSerialize();

View file

@ -14,13 +14,14 @@
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
namespace Server;
public class Persistence
public static class Persistence
{
public const int DefaultPriority = 100;
@ -30,7 +31,7 @@ public class Persistence
string name,
Action serializer,
Action<string> snapshotWriter,
Action<string> deserializer,
Action<string, Dictionary<ulong, string>> deserializer,
int priority = DefaultPriority
)
{
@ -50,24 +51,79 @@ public class Persistence
public static void Load(string path)
{
var typesDb = LoadTypes(path);
// This should probably not be parallel since Mobiles must be loaded before Items
foreach (var entry in _registry)
{
entry.Deserialize(path);
entry.Deserialize(path, typesDb);
}
}
private static Dictionary<ulong, string> LoadTypes(string path)
{
var db = new Dictionary<ulong, string>();
string tdbPath = Path.Combine(path, "SerializedTypes.db");
if (!File.Exists(tdbPath))
{
return db;
}
using FileStream tdb = new FileStream(tdbPath, FileMode.Open, FileAccess.Read, FileShare.Read);
BinaryReader tdbReader = new BinaryReader(tdb);
var version = tdbReader.ReadInt32();
var count = tdbReader.ReadInt32();
for (var i = 0; i < count; ++i)
{
var hash = tdbReader.ReadUInt64();
var typeName = tdbReader.ReadString();
db[hash] = typeName;
}
return db;
}
public static void Serialize()
{
Parallel.ForEach(_registry, entry => entry.Serialize());
}
public static void WriteSnapshot(string path)
public static void WriteSnapshot(string path, ConcurrentQueue<Type> types)
{
foreach (var entry in _registry)
{
entry.WriteSnapshot(path);
}
// Dedupe the queue.
foreach (var type in types)
{
_typesSet.Add(type);
}
WriteSerializedTypesSnapshot(path, _typesSet);
_typesSet.Clear();
}
private static HashSet<Type> _typesSet = new();
public static void WriteSerializedTypesSnapshot(string path, HashSet<Type> types)
{
string tdbPath = Path.Combine(path, "SerializedTypes.db");
using var tdb = new BinaryFileWriter(tdbPath, false);
tdb.Write(0); // version
tdb.Write(types.Count);
foreach (var type in types)
{
var fullName = type.FullName;
tdb.Write(HashUtility.ComputeHash64(fullName));
tdb.Write(fullName);
}
}
public record RegistryEntry
@ -76,7 +132,7 @@ public class Persistence
public int Priority { get; init; }
public Action Serialize { get; init; } // Serializing to memory buffers
public Action<string> WriteSnapshot { get; init; }
public Action<string> Deserialize { get; init; }
public Action<string, Dictionary<ulong, string>> Deserialize { get; init; }
}
internal class RegistryEntryComparer : IComparer<RegistryEntry>