warrentyexpired/ModernUO
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Adds ReadType/Write(Type) and improves type referencing (#1172)

Browse source 
## Changes
* Improves type hashing by introducing xxHash3 (64bit)
* Removes individual `tdb` files in favor of a single `SerializedTypes.db` file. This file is only used to identify a type that is being deserialized, which doesn't exist.
* Adds duplicate type alias detection
* Adds `AssemblyHandler.FindTypeByHash`

View changed files whitespaces: https://github.com/modernuo/ModernUO/pull/1172/files?diff=split&w=1

## SerializedTypes.db
The serialized types file is used to get back the original name of a type in case it no longer exists in code. This can easily be necessary if a class is renamed in code and no `TypeAlias` is provided.

### Format
byte[4] - version
byte[4] - count
--array--
byte[8] - xxHash
byte[1] - flag, 0 - null, 1 - not null
byte[n] - Full class name in UTF8

### Example
<img width="472" alt="SerializedTypes_Example" src="https://user-images.githubusercontent.com/3953314/195255429-31d24293-6bd1-419e-811b-07874dd0f78d.png">

## Benchmarks
Serialized 500 Type fields. The 8192bytes comes from the _ConcurrentQueue_ that would later be used for SerializedTypes.
Note that the queue is never cleared, so it's size grew considerably.
```cs
|               Method |     Mean |    Error |   StdDev | Allocated |
|--------------------- |---------:|---------:|---------:|----------:|
|      BenchmarkXXHash | 18.44 us | 0.278 us | 0.260 us |    8192 B |
| BenchmarkTypeStrings | 25.09 us | 0.292 us | 0.259 us |         - |
```

TODO:
* Add support in the Serialization Generator for `ReadType()` and `Write(Type)`
* Remove `SetTypeRef` from Serialization Generator
This commit is contained in:
Kamron Batman 2022-10-11 22:17:22 -07:00 • committed by GitHub
parent f268d5d4e2
commit e1e30998ba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 616 additions and 291 deletions
Download patch file Download diff file Expand all files Collapse all files

59
Projects/Server/Serialization/BufferWriter.cs
View file

@ -2,7 +2,7 @@
* ModernUO *
* Copyright 2019-2022 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: BufferedFileWriter.cs *
* File: BufferWriter.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
@ -14,6 +14,7 @@
*************************************************************************/
using System;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
@ -25,8 +26,9 @@ namespace Server;
public class BufferWriter : IGenericWriter
{
private readonly Encoding m_Encoding;
private readonly bool m_PrefixStrings;
private ConcurrentQueue<Type> _types;
private Encoding _encoding;
private bool _prefixStrings;
private long _bytesWritten;
private long _index;
@ -53,22 +55,24 @@ public class BufferWriter : IGenericWriter
private byte[] _buffer;
public BufferWriter(byte[] buffer, bool prefixStr)
public BufferWriter(byte[] buffer, bool prefixStr, ConcurrentQueue<Type> types = null)
{
m_PrefixStrings = prefixStr;
m_Encoding = TextEncoding.UTF8;
_prefixStrings = prefixStr;
_encoding = TextEncoding.UTF8;
_buffer = buffer;
_types = types;
}
public BufferWriter(bool prefixStr) : this(0, prefixStr)
public BufferWriter(bool prefixStr, ConcurrentQueue<Type> types = null) : this(0, prefixStr, types)
{
}
public BufferWriter(int count, bool prefixStr)
public BufferWriter(int count, bool prefixStr, ConcurrentQueue<Type> types = null)
{
m_PrefixStrings = prefixStr;
m_Encoding = TextEncoding.UTF8;
_prefixStrings = prefixStr;
_encoding = TextEncoding.UTF8;
_buffer = GC.AllocateUninitializedArray<byte>(count < 1 ? BufferSize : count);
_types = types;
}
public virtual long Position => Index;
@ -135,6 +139,7 @@ public class BufferWriter : IGenericWriter
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(BitArray bitArray)
{
var byteLength = BitArray.GetByteArrayLengthFromBitLength(bitArray.Length);
@ -145,6 +150,7 @@ public class BufferWriter : IGenericWriter
Index += byteLength;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public virtual long Seek(long offset, SeekOrigin origin)
{
Debug.Assert(
@ -168,9 +174,10 @@ public class BufferWriter : IGenericWriter
});
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(string value)
{
if (m_PrefixStrings)
if (_prefixStrings)
{
if (value == null)
{
@ -188,6 +195,7 @@ public class BufferWriter : IGenericWriter
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(long value)
{
FlushIfNeeded(8);
@ -202,6 +210,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 56);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(ulong value)
{
FlushIfNeeded(8);
@ -216,6 +225,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 56);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(int value)
{
FlushIfNeeded(4);
@ -226,6 +236,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 24);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(uint value)
{
FlushIfNeeded(4);
@ -236,6 +247,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 24);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(short value)
{
FlushIfNeeded(2);
@ -244,6 +256,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 8);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(ushort value)
{
FlushIfNeeded(2);
@ -252,6 +265,7 @@ public class BufferWriter : IGenericWriter
_buffer[Index++] = (byte)(value >> 8);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe void Write(double value)
{
FlushIfNeeded(8);
@ -264,6 +278,7 @@ public class BufferWriter : IGenericWriter
Index += 8;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe void Write(float value)
{
FlushIfNeeded(4);
@ -300,9 +315,25 @@ public class BufferWriter : IGenericWriter
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(Serial serial) => Write(serial.Value);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(Type type)
{
if (type == null)
{
Write((byte)0);
}
else
{
Write((byte)0x2); // xxHash3 64bit
Write(AssemblyHandler.GetTypeHash(type));
_types?.Enqueue(type);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal void InternalWriteString(string value)
{
var remaining = m_Encoding.GetByteCount(value);
var remaining = _encoding.GetByteCount(value);
((IGenericWriter)this).WriteEncodedInt(remaining);
@ -313,14 +344,14 @@ public class BufferWriter : IGenericWriter
// It is much faster to encode to stack buffer, then copy to the real buffer
Span<byte> span = stackalloc byte[Math.Min(BufferSize, 256)];
var maxChars = span.Length / m_Encoding.GetMaxByteCount(1);
var maxChars = span.Length / _encoding.GetMaxByteCount(1);
var charsLeft = value.Length;
var current = 0;
while (charsLeft > 0)
{
var charCount = Math.Min(charsLeft, maxChars);
var bytesWritten = m_Encoding.GetBytes(value.AsSpan(current, charCount), span);
var bytesWritten = _encoding.GetBytes(value.AsSpan(current, charCount), span);
remaining -= bytesWritten;
charsLeft -= charCount;
current += charCount;