fix: Adds ReadType/Write(Type) and improves type referencing (#1172)
## Changes * Improves type hashing by introducing xxHash3 (64bit) * Removes individual `tdb` files in favor of a single `SerializedTypes.db` file. This file is only used to identify a type that is being deserialized, which doesn't exist. * Adds duplicate type alias detection * Adds `AssemblyHandler.FindTypeByHash` View changed files whitespaces: https://github.com/modernuo/ModernUO/pull/1172/files?diff=split&w=1 ## SerializedTypes.db The serialized types file is used to get back the original name of a type in case it no longer exists in code. This can easily be necessary if a class is renamed in code and no `TypeAlias` is provided. ### Format byte[4] - version byte[4] - count --array-- byte[8] - xxHash byte[1] - flag, 0 - null, 1 - not null byte[n] - Full class name in UTF8 ### Example <img width="472" alt="SerializedTypes_Example" src="https://user-images.githubusercontent.com/3953314/195255429-31d24293-6bd1-419e-811b-07874dd0f78d.png"> ## Benchmarks Serialized 500 Type fields. The 8192bytes comes from the _ConcurrentQueue_ that would later be used for SerializedTypes. Note that the queue is never cleared, so it's size grew considerably. ```cs | Method | Mean | Error | StdDev | Allocated | |--------------------- |---------:|---------:|---------:|----------:| | BenchmarkXXHash | 18.44 us | 0.278 us | 0.260 us | 8192 B | | BenchmarkTypeStrings | 25.09 us | 0.292 us | 0.259 us | - | ``` TODO: * Add support in the Serialization Generator for `ReadType()` and `Write(Type)` * Remove `SetTypeRef` from Serialization Generator
This commit is contained in:
parent
f268d5d4e2
commit
e1e30998ba
28 changed files with 616 additions and 291 deletions
|
|
@ -14,6 +14,8 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using System.Threading.Tasks;
|
||||
|
|
@ -27,30 +29,53 @@ public static class AdhocPersistence
|
|||
* Note: The buffer may not be the same after returning from the function if more data is written
|
||||
* than the initial buffer can handle.
|
||||
*/
|
||||
public static BufferWriter Serialize(Action<IGenericWriter> serializer)
|
||||
public static BufferWriter Serialize(Action<IGenericWriter> serializer, ConcurrentQueue<Type> types)
|
||||
{
|
||||
var saveBuffer = new BufferWriter(true);
|
||||
var saveBuffer = new BufferWriter(true, types);
|
||||
serializer(saveBuffer);
|
||||
return saveBuffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a buffer to disk. This function should be called asynchronously.
|
||||
* Writes the filePath for the binary data, and an accompanying SerializedTypes.db file of all possible types.
|
||||
*/
|
||||
public static void WriteSnapshot(string filePath, Span<byte> buffer)
|
||||
public static void WriteSnapshot(FileInfo file, Span<byte> buffer)
|
||||
{
|
||||
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
|
||||
var file = new FileInfo(fullPath);
|
||||
PathUtility.EnsureDirectory(file.DirectoryName);
|
||||
var dirPath = file.DirectoryName;
|
||||
PathUtility.EnsureDirectory(dirPath);
|
||||
|
||||
using var fs = new FileStream(fullPath, FileMode.Create, FileAccess.Write);
|
||||
using var fs = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
|
||||
fs.Write(buffer);
|
||||
}
|
||||
|
||||
public static void SerializeAndSnapshot(string filePath, Action<IGenericWriter> serializer)
|
||||
/**
|
||||
* Serializes to a memory buffer synchronously, then flushes to the path asynchronously.
|
||||
* See WriteSnapshot for more info about how the snapshot.
|
||||
*/
|
||||
public static void SerializeAndSnapshot(string filePath, Action<IGenericWriter> serializer, ConcurrentQueue<Type> types = null)
|
||||
{
|
||||
var saveBuffer = Serialize(serializer);
|
||||
Task.Run(() => { WriteSnapshot(filePath, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position)); });
|
||||
types ??= new ConcurrentQueue<Type>();
|
||||
var saveBuffer = Serialize(serializer, types);
|
||||
Task.Run(
|
||||
() =>
|
||||
{
|
||||
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
|
||||
var file = new FileInfo(fullPath);
|
||||
|
||||
WriteSnapshot(file, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position));
|
||||
|
||||
// TODO: Create a PooledHashSet if performance becomes an issue.
|
||||
var typesSet = new HashSet<Type>();
|
||||
|
||||
// Dedupe the queue.
|
||||
foreach (var type in types)
|
||||
{
|
||||
typesSet.Add(type);
|
||||
}
|
||||
|
||||
Persistence.WriteSerializedTypesSnapshot(file.DirectoryName, typesSet);
|
||||
});
|
||||
}
|
||||
|
||||
public static void Deserialize(string filePath, Action<IGenericReader> deserializer)
|
||||
|
|
|
|||
|
|
@ -14,27 +14,34 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using Server.Buffers;
|
||||
using Server.Collections;
|
||||
using Server.Logging;
|
||||
using Server.Text;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public class BinaryFileReader : IGenericReader, IDisposable
|
||||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(BinaryFileReader));
|
||||
|
||||
private Dictionary<ulong, string> _typesDb;
|
||||
private BinaryReader _reader;
|
||||
private Encoding _encoding;
|
||||
|
||||
public BinaryFileReader(BinaryReader br, Encoding encoding = null)
|
||||
public BinaryFileReader(BinaryReader br, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
|
||||
{
|
||||
_reader = br;
|
||||
_encoding = encoding ?? TextEncoding.UTF8;
|
||||
_typesDb = typesDb;
|
||||
}
|
||||
|
||||
public BinaryFileReader(Stream stream, Encoding encoding = null) : this(new BinaryReader(stream), encoding)
|
||||
public BinaryFileReader(Stream stream, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
|
||||
: this(new BinaryReader(stream), typesDb, encoding)
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -46,23 +53,20 @@ public class BinaryFileReader : IGenericReader, IDisposable
|
|||
public DateTime LastSerialized { get; init; }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadString(bool intern = false)
|
||||
{
|
||||
if (!ReadBool())
|
||||
{
|
||||
return null;
|
||||
}
|
||||
public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null;
|
||||
|
||||
public string ReadStringRaw(bool intern = false)
|
||||
{
|
||||
var length = ((IGenericReader)this).ReadEncodedInt();
|
||||
if (length <= 0)
|
||||
{
|
||||
return intern ? Utility.Intern("") : "";
|
||||
return "".Intern();
|
||||
}
|
||||
|
||||
byte[] buffer = STArrayPool<byte>.Shared.Rent(length);
|
||||
var str = TextEncoding.GetString(buffer.AsSpan(0, length), _encoding);
|
||||
STArrayPool<byte>.Shared.Return(buffer);
|
||||
return intern ? Utility.Intern(str) : str;
|
||||
return intern ? str.Intern() : str;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
|
|
@ -101,6 +105,62 @@ public class BinaryFileReader : IGenericReader, IDisposable
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Serial ReadSerial() => (Serial)_reader.ReadUInt32();
|
||||
|
||||
public Type ReadType() =>
|
||||
ReadByte() switch
|
||||
{
|
||||
0 => null,
|
||||
1 => AssemblyHandler.FindTypeByFullName(ReadStringRaw()), // Backward compatibility
|
||||
2 => ReadTypeByHash()
|
||||
};
|
||||
|
||||
public Type ReadTypeByHash()
|
||||
{
|
||||
var hash = ReadULong();
|
||||
var t = AssemblyHandler.FindTypeByHash(hash);
|
||||
|
||||
if (t != null)
|
||||
{
|
||||
return t;
|
||||
}
|
||||
|
||||
if (_typesDb == null)
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"The file SerializedTypes.db was not loaded. Type hash '{hash}' could not be found."),
|
||||
"Invalid {Hash} at position {Position}",
|
||||
hash,
|
||||
Position
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!_typesDb.TryGetValue(hash, out var typeName))
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"Type hash '{hash}' is not present in the serialized types database."),
|
||||
"Invalid type hash {Hash} at position {Position}",
|
||||
hash,
|
||||
Position
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
t = AssemblyHandler.FindTypeByFullName(typeName, false);
|
||||
|
||||
if (t == null)
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"Type '{typeName}' was not found."),
|
||||
"Type {Type} was not found.",
|
||||
typeName
|
||||
);
|
||||
}
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int Read(Span<byte> buffer) => _reader.Read(buffer);
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
|
||||
namespace Server;
|
||||
|
|
@ -23,11 +24,11 @@ public class BinaryFileWriter : BufferWriter, IDisposable
|
|||
private readonly Stream _file;
|
||||
private long _position;
|
||||
|
||||
public BinaryFileWriter(string filename, bool prefixStr) :
|
||||
this(new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None), prefixStr)
|
||||
public BinaryFileWriter(string filename, bool prefixStr, ConcurrentQueue<Type> types = null) :
|
||||
this(new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None), prefixStr, types)
|
||||
{}
|
||||
|
||||
public BinaryFileWriter(Stream stream, bool prefixStr) : base(prefixStr)
|
||||
public BinaryFileWriter(Stream stream, bool prefixStr, ConcurrentQueue<Type> types = null) : base(prefixStr, types)
|
||||
{
|
||||
_file = stream;
|
||||
_position = _file.Position;
|
||||
|
|
|
|||
|
|
@ -15,30 +15,40 @@
|
|||
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using Server.Collections;
|
||||
using Server.Logging;
|
||||
using Server.Text;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public class BufferReader : IGenericReader
|
||||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(BufferReader));
|
||||
|
||||
private Dictionary<ulong, string> _typesDb;
|
||||
private Encoding _encoding;
|
||||
private byte[] _buffer;
|
||||
private int _position;
|
||||
|
||||
public long Position => _position;
|
||||
|
||||
public BufferReader(byte[] buffer, Encoding encoding = null)
|
||||
public BufferReader(byte[] buffer, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
|
||||
{
|
||||
_buffer = buffer;
|
||||
_encoding = encoding ?? TextEncoding.UTF8;
|
||||
_typesDb = typesDb;
|
||||
}
|
||||
|
||||
public BufferReader(byte[] buffer, DateTime lastSerialized) : this(buffer) => LastSerialized = lastSerialized;
|
||||
public BufferReader(byte[] buffer, DateTime lastSerialized, Dictionary<ulong, string> typesDb = null) : this(buffer)
|
||||
{
|
||||
LastSerialized = lastSerialized;
|
||||
_typesDb = typesDb;
|
||||
}
|
||||
|
||||
public void Reset(byte[] newBuffer, out byte[] oldBuffer)
|
||||
{
|
||||
|
|
@ -49,22 +59,20 @@ public class BufferReader : IGenericReader
|
|||
|
||||
public DateTime LastSerialized { get; init; }
|
||||
|
||||
public string ReadString(bool intern = false)
|
||||
{
|
||||
if (!ReadBool())
|
||||
{
|
||||
return null;
|
||||
}
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null;
|
||||
|
||||
public string ReadStringRaw(bool intern = false)
|
||||
{
|
||||
var length = ((IGenericReader)this).ReadEncodedInt();
|
||||
if (length <= 0)
|
||||
{
|
||||
return intern ? Utility.Intern("") : "";
|
||||
return "".Intern();
|
||||
}
|
||||
|
||||
var str = TextEncoding.GetString(_buffer.AsSpan(_position, length), _encoding);
|
||||
_position += length;
|
||||
return intern ? Utility.Intern(str) : str;
|
||||
return intern ? str.Intern() : str;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
|
|
@ -143,6 +151,62 @@ public class BufferReader : IGenericReader
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Serial ReadSerial() => (Serial)ReadUInt();
|
||||
|
||||
public Type ReadType() =>
|
||||
ReadByte() switch
|
||||
{
|
||||
0 => null,
|
||||
1 => AssemblyHandler.FindTypeByFullName(ReadStringRaw()), // Backward compatibility
|
||||
2 => ReadTypeByHash()
|
||||
};
|
||||
|
||||
public Type ReadTypeByHash()
|
||||
{
|
||||
var hash = ReadULong();
|
||||
var t = AssemblyHandler.FindTypeByHash(hash);
|
||||
|
||||
if (t != null)
|
||||
{
|
||||
return t;
|
||||
}
|
||||
|
||||
if (_typesDb == null)
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"The file SerializedTypes.db was not loaded. Type hash '{hash}' could not be found."),
|
||||
"Invalid {Hash} at position {Position}",
|
||||
hash,
|
||||
Position
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!_typesDb.TryGetValue(hash, out var typeName))
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"Type hash '{hash}' is not present in the serialized types database."),
|
||||
"Invalid type hash {Hash} at position {Position}",
|
||||
hash,
|
||||
Position
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
t = AssemblyHandler.FindTypeByFullName(typeName, false);
|
||||
|
||||
if (t == null)
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"Type '{typeName}' was not found."),
|
||||
"Type {Type} was not found.",
|
||||
typeName
|
||||
);
|
||||
}
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
public int Read(Span<byte> buffer)
|
||||
{
|
||||
var length = buffer.Length;
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* ModernUO *
|
||||
* Copyright 2019-2022 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: BufferedFileWriter.cs *
|
||||
* File: BufferWriter.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
|
|
@ -14,6 +14,7 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
|
@ -25,8 +26,9 @@ namespace Server;
|
|||
|
||||
public class BufferWriter : IGenericWriter
|
||||
{
|
||||
private readonly Encoding m_Encoding;
|
||||
private readonly bool m_PrefixStrings;
|
||||
private ConcurrentQueue<Type> _types;
|
||||
private Encoding _encoding;
|
||||
private bool _prefixStrings;
|
||||
private long _bytesWritten;
|
||||
private long _index;
|
||||
|
||||
|
|
@ -53,22 +55,24 @@ public class BufferWriter : IGenericWriter
|
|||
|
||||
private byte[] _buffer;
|
||||
|
||||
public BufferWriter(byte[] buffer, bool prefixStr)
|
||||
public BufferWriter(byte[] buffer, bool prefixStr, ConcurrentQueue<Type> types = null)
|
||||
{
|
||||
m_PrefixStrings = prefixStr;
|
||||
m_Encoding = TextEncoding.UTF8;
|
||||
_prefixStrings = prefixStr;
|
||||
_encoding = TextEncoding.UTF8;
|
||||
_buffer = buffer;
|
||||
_types = types;
|
||||
}
|
||||
|
||||
public BufferWriter(bool prefixStr) : this(0, prefixStr)
|
||||
public BufferWriter(bool prefixStr, ConcurrentQueue<Type> types = null) : this(0, prefixStr, types)
|
||||
{
|
||||
}
|
||||
|
||||
public BufferWriter(int count, bool prefixStr)
|
||||
public BufferWriter(int count, bool prefixStr, ConcurrentQueue<Type> types = null)
|
||||
{
|
||||
m_PrefixStrings = prefixStr;
|
||||
m_Encoding = TextEncoding.UTF8;
|
||||
_prefixStrings = prefixStr;
|
||||
_encoding = TextEncoding.UTF8;
|
||||
_buffer = GC.AllocateUninitializedArray<byte>(count < 1 ? BufferSize : count);
|
||||
_types = types;
|
||||
}
|
||||
|
||||
public virtual long Position => Index;
|
||||
|
|
@ -135,6 +139,7 @@ public class BufferWriter : IGenericWriter
|
|||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(BitArray bitArray)
|
||||
{
|
||||
var byteLength = BitArray.GetByteArrayLengthFromBitLength(bitArray.Length);
|
||||
|
|
@ -145,6 +150,7 @@ public class BufferWriter : IGenericWriter
|
|||
Index += byteLength;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public virtual long Seek(long offset, SeekOrigin origin)
|
||||
{
|
||||
Debug.Assert(
|
||||
|
|
@ -168,9 +174,10 @@ public class BufferWriter : IGenericWriter
|
|||
});
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(string value)
|
||||
{
|
||||
if (m_PrefixStrings)
|
||||
if (_prefixStrings)
|
||||
{
|
||||
if (value == null)
|
||||
{
|
||||
|
|
@ -188,6 +195,7 @@ public class BufferWriter : IGenericWriter
|
|||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(long value)
|
||||
{
|
||||
FlushIfNeeded(8);
|
||||
|
|
@ -202,6 +210,7 @@ public class BufferWriter : IGenericWriter
|
|||
_buffer[Index++] = (byte)(value >> 56);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(ulong value)
|
||||
{
|
||||
FlushIfNeeded(8);
|
||||
|
|
@ -216,6 +225,7 @@ public class BufferWriter : IGenericWriter
|
|||
_buffer[Index++] = (byte)(value >> 56);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(int value)
|
||||
{
|
||||
FlushIfNeeded(4);
|
||||
|
|
@ -226,6 +236,7 @@ public class BufferWriter : IGenericWriter
|
|||
_buffer[Index++] = (byte)(value >> 24);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(uint value)
|
||||
{
|
||||
FlushIfNeeded(4);
|
||||
|
|
@ -236,6 +247,7 @@ public class BufferWriter : IGenericWriter
|
|||
_buffer[Index++] = (byte)(value >> 24);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(short value)
|
||||
{
|
||||
FlushIfNeeded(2);
|
||||
|
|
@ -244,6 +256,7 @@ public class BufferWriter : IGenericWriter
|
|||
_buffer[Index++] = (byte)(value >> 8);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(ushort value)
|
||||
{
|
||||
FlushIfNeeded(2);
|
||||
|
|
@ -252,6 +265,7 @@ public class BufferWriter : IGenericWriter
|
|||
_buffer[Index++] = (byte)(value >> 8);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public unsafe void Write(double value)
|
||||
{
|
||||
FlushIfNeeded(8);
|
||||
|
|
@ -264,6 +278,7 @@ public class BufferWriter : IGenericWriter
|
|||
Index += 8;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public unsafe void Write(float value)
|
||||
{
|
||||
FlushIfNeeded(4);
|
||||
|
|
@ -300,9 +315,25 @@ public class BufferWriter : IGenericWriter
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(Serial serial) => Write(serial.Value);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(Type type)
|
||||
{
|
||||
if (type == null)
|
||||
{
|
||||
Write((byte)0);
|
||||
}
|
||||
else
|
||||
{
|
||||
Write((byte)0x2); // xxHash3 64bit
|
||||
Write(AssemblyHandler.GetTypeHash(type));
|
||||
_types?.Enqueue(type);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal void InternalWriteString(string value)
|
||||
{
|
||||
var remaining = m_Encoding.GetByteCount(value);
|
||||
var remaining = _encoding.GetByteCount(value);
|
||||
|
||||
((IGenericWriter)this).WriteEncodedInt(remaining);
|
||||
|
||||
|
|
@ -313,14 +344,14 @@ public class BufferWriter : IGenericWriter
|
|||
|
||||
// It is much faster to encode to stack buffer, then copy to the real buffer
|
||||
Span<byte> span = stackalloc byte[Math.Min(BufferSize, 256)];
|
||||
var maxChars = span.Length / m_Encoding.GetMaxByteCount(1);
|
||||
var maxChars = span.Length / _encoding.GetMaxByteCount(1);
|
||||
var charsLeft = value.Length;
|
||||
var current = 0;
|
||||
|
||||
while (charsLeft > 0)
|
||||
{
|
||||
var charCount = Math.Min(charsLeft, maxChars);
|
||||
var bytesWritten = m_Encoding.GetBytes(value.AsSpan(current, charCount), span);
|
||||
var bytesWritten = _encoding.GetBytes(value.AsSpan(current, charCount), span);
|
||||
remaining -= bytesWritten;
|
||||
charsLeft -= charCount;
|
||||
current += charCount;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
namespace Server;
|
||||
|
|
@ -31,22 +33,22 @@ public static class GenericPersistence
|
|||
|
||||
void Serialize()
|
||||
{
|
||||
saveBuffer ??= new BufferWriter(true);
|
||||
saveBuffer ??= new BufferWriter(true, World.SerializedTypes);
|
||||
saveBuffer.Seek(0, SeekOrigin.Begin);
|
||||
|
||||
serializer(saveBuffer);
|
||||
}
|
||||
|
||||
void WriterSnapshot(string savePath)
|
||||
void WriteSnapshot(string savePath)
|
||||
{
|
||||
string binPath = Path.Combine(savePath, name, $"{name}.bin");
|
||||
var buffer = saveBuffer!.Buffer.AsSpan(0, (int)saveBuffer.Position);
|
||||
AdhocPersistence.WriteSnapshot(binPath, buffer);
|
||||
AdhocPersistence.WriteSnapshot(new FileInfo(binPath), buffer);
|
||||
}
|
||||
|
||||
void Deserialize(string savePath) =>
|
||||
void Deserialize(string savePath, Dictionary<ulong, string> typesDb) =>
|
||||
AdhocPersistence.Deserialize(Path.Combine(savePath, name, $"{name}.bin"), deserializer);
|
||||
|
||||
Persistence.Register(name, Serialize, WriterSnapshot, Deserialize, priority);
|
||||
Persistence.Register(name, Serialize, WriteSnapshot, Deserialize, priority);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ public interface IGenericReader
|
|||
sbyte ReadSByte();
|
||||
bool ReadBool();
|
||||
Serial ReadSerial();
|
||||
Type ReadType();
|
||||
|
||||
DateTime ReadDateTime() => new(ReadLong(), DateTimeKind.Utc);
|
||||
TimeSpan ReadTimeSpan() => new(ReadLong());
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ public interface IGenericWriter
|
|||
void Write(sbyte value);
|
||||
void Write(bool value);
|
||||
void Write(Serial serial);
|
||||
void Write(Type type);
|
||||
|
||||
void Write(DateTime value)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
|
||||
namespace Server;
|
||||
|
|
@ -28,7 +29,6 @@ public interface ISerializable
|
|||
long SavePosition { get; protected internal set; }
|
||||
BufferWriter SaveBuffer { get; protected internal set; }
|
||||
|
||||
int TypeRef { get; }
|
||||
Serial Serial { get; }
|
||||
|
||||
// Executed on every entity, before it's serialized.
|
||||
|
|
@ -39,11 +39,9 @@ public interface ISerializable
|
|||
void Delete();
|
||||
bool Deleted { get; }
|
||||
|
||||
void SetTypeRef(Type type);
|
||||
|
||||
public void InitializeSaveBuffer(byte[] buffer)
|
||||
public void InitializeSaveBuffer(byte[] buffer, ConcurrentQueue<Type> types)
|
||||
{
|
||||
SaveBuffer = new BufferWriter(buffer, true);
|
||||
SaveBuffer = new BufferWriter(buffer, true, types);
|
||||
if (World.DirtyTrackingEnabled)
|
||||
{
|
||||
SavePosition = SaveBuffer.Position;
|
||||
|
|
@ -54,9 +52,9 @@ public interface ISerializable
|
|||
}
|
||||
}
|
||||
|
||||
public void Serialize()
|
||||
public void Serialize(ConcurrentQueue<Type> types)
|
||||
{
|
||||
SaveBuffer ??= new BufferWriter(true);
|
||||
SaveBuffer ??= new BufferWriter(true, types);
|
||||
|
||||
BeforeSerialize();
|
||||
|
||||
|
|
|
|||
|
|
@ -14,13 +14,14 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public class Persistence
|
||||
public static class Persistence
|
||||
{
|
||||
public const int DefaultPriority = 100;
|
||||
|
||||
|
|
@ -30,7 +31,7 @@ public class Persistence
|
|||
string name,
|
||||
Action serializer,
|
||||
Action<string> snapshotWriter,
|
||||
Action<string> deserializer,
|
||||
Action<string, Dictionary<ulong, string>> deserializer,
|
||||
int priority = DefaultPriority
|
||||
)
|
||||
{
|
||||
|
|
@ -50,24 +51,79 @@ public class Persistence
|
|||
|
||||
public static void Load(string path)
|
||||
{
|
||||
var typesDb = LoadTypes(path);
|
||||
|
||||
// This should probably not be parallel since Mobiles must be loaded before Items
|
||||
foreach (var entry in _registry)
|
||||
{
|
||||
entry.Deserialize(path);
|
||||
entry.Deserialize(path, typesDb);
|
||||
}
|
||||
}
|
||||
|
||||
private static Dictionary<ulong, string> LoadTypes(string path)
|
||||
{
|
||||
var db = new Dictionary<ulong, string>();
|
||||
|
||||
string tdbPath = Path.Combine(path, "SerializedTypes.db");
|
||||
if (!File.Exists(tdbPath))
|
||||
{
|
||||
return db;
|
||||
}
|
||||
|
||||
using FileStream tdb = new FileStream(tdbPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
BinaryReader tdbReader = new BinaryReader(tdb);
|
||||
|
||||
var version = tdbReader.ReadInt32();
|
||||
var count = tdbReader.ReadInt32();
|
||||
|
||||
for (var i = 0; i < count; ++i)
|
||||
{
|
||||
var hash = tdbReader.ReadUInt64();
|
||||
var typeName = tdbReader.ReadString();
|
||||
db[hash] = typeName;
|
||||
}
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
public static void Serialize()
|
||||
{
|
||||
Parallel.ForEach(_registry, entry => entry.Serialize());
|
||||
}
|
||||
|
||||
public static void WriteSnapshot(string path)
|
||||
public static void WriteSnapshot(string path, ConcurrentQueue<Type> types)
|
||||
{
|
||||
foreach (var entry in _registry)
|
||||
{
|
||||
entry.WriteSnapshot(path);
|
||||
}
|
||||
|
||||
// Dedupe the queue.
|
||||
foreach (var type in types)
|
||||
{
|
||||
_typesSet.Add(type);
|
||||
}
|
||||
|
||||
WriteSerializedTypesSnapshot(path, _typesSet);
|
||||
_typesSet.Clear();
|
||||
}
|
||||
|
||||
private static HashSet<Type> _typesSet = new();
|
||||
|
||||
public static void WriteSerializedTypesSnapshot(string path, HashSet<Type> types)
|
||||
{
|
||||
string tdbPath = Path.Combine(path, "SerializedTypes.db");
|
||||
using var tdb = new BinaryFileWriter(tdbPath, false);
|
||||
|
||||
tdb.Write(0); // version
|
||||
tdb.Write(types.Count);
|
||||
|
||||
foreach (var type in types)
|
||||
{
|
||||
var fullName = type.FullName;
|
||||
tdb.Write(HashUtility.ComputeHash64(fullName));
|
||||
tdb.Write(fullName);
|
||||
}
|
||||
}
|
||||
|
||||
public record RegistryEntry
|
||||
|
|
@ -76,7 +132,7 @@ public class Persistence
|
|||
public int Priority { get; init; }
|
||||
public Action Serialize { get; init; } // Serializing to memory buffers
|
||||
public Action<string> WriteSnapshot { get; init; }
|
||||
public Action<string> Deserialize { get; init; }
|
||||
public Action<string, Dictionary<ulong, string>> Deserialize { get; init; }
|
||||
}
|
||||
|
||||
internal class RegistryEntryComparer : IComparer<RegistryEntry>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue