fix(core): Serialization issues (#350)

- [X] Fixes IPAddresses not having enough space
- [X] Streamlines some code
- [X] Fixes base escortables to not load destination tables in a dangerous way.
- [X] Fixes some bad assumptions about resizing buffers
- [X] Fixes little endian issue with circular buffer writer

This doesn't seem to address #349. That issue requires more investigation because I am not seeing where the issue would be.

Closes #347
Closes #322
This commit is contained in:
Kamron Batman 2020-12-13 13:15:48 -08:00 committed by GitHub
parent 3acb1414dd
commit e17413af21
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 186 additions and 222 deletions

View file

@ -88,7 +88,11 @@ namespace System.Buffers
{
if (!BinaryPrimitives.TryWriteInt16BigEndian(_first.Slice(Position), value))
{
// Not enough space. Split the spans
if (BitConverter.IsLittleEndian)
{
value = BinaryPrimitives.ReverseEndianness(value);
}
Write((byte)(value >> 8));
Write((byte)value);
}
@ -114,7 +118,11 @@ namespace System.Buffers
{
if (!BinaryPrimitives.TryWriteUInt16BigEndian(_first.Slice(Position), value))
{
// Not enough space. Split the spans
if (BitConverter.IsLittleEndian)
{
value = BinaryPrimitives.ReverseEndianness(value);
}
Write((byte)(value >> 8));
Write((byte)value);
}
@ -140,7 +148,11 @@ namespace System.Buffers
{
if (!BinaryPrimitives.TryWriteInt32BigEndian(_first.Slice(Position), value))
{
// Not enough space. Split the spans
if (BitConverter.IsLittleEndian)
{
value = BinaryPrimitives.ReverseEndianness(value);
}
Write((byte)(value >> 24));
Write((byte)(value >> 16));
Write((byte)(value >> 8));
@ -168,7 +180,11 @@ namespace System.Buffers
{
if (!BinaryPrimitives.TryWriteInt32LittleEndian(_first.Slice(Position), value))
{
// Not enough space. Split the spans
if (!BitConverter.IsLittleEndian)
{
value = BinaryPrimitives.ReverseEndianness(value);
}
Write((byte)(value >> 24));
Write((byte)(value >> 16));
Write((byte)(value >> 8));
@ -199,7 +215,11 @@ namespace System.Buffers
{
if (!BinaryPrimitives.TryWriteUInt32BigEndian(_first.Slice(Position), value))
{
// Not enough space. Split the spans
if (BitConverter.IsLittleEndian)
{
value = BinaryPrimitives.ReverseEndianness(value);
}
Write((byte)(value >> 24));
Write((byte)(value >> 16));
Write((byte)(value >> 8));
@ -220,6 +240,38 @@ namespace System.Buffers
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteLE(uint value)
{
if (Position < _first.Length)
{
if (!BinaryPrimitives.TryWriteUInt32LittleEndian(_first.Slice(Position), value))
{
if (!BitConverter.IsLittleEndian)
{
value = BinaryPrimitives.ReverseEndianness(value);
}
Write((byte)(value >> 24));
Write((byte)(value >> 16));
Write((byte)(value >> 8));
Write((byte)value);
}
else
{
Position += 4;
}
}
else if (BinaryPrimitives.TryWriteUInt32LittleEndian(_second.Slice(Position - _first.Length), value))
{
Position += 4;
}
else
{
throw new OutOfMemoryException();
}
}
/// <summary>
/// Writes a 8-byte signed integer value to the underlying stream.
/// </summary>
@ -230,7 +282,11 @@ namespace System.Buffers
{
if (!BinaryPrimitives.TryWriteInt64BigEndian(_first.Slice(Position), value))
{
// Not enough space. Split the spans
if (BitConverter.IsLittleEndian)
{
value = BinaryPrimitives.ReverseEndianness(value);
}
Write((byte)(value >> 56));
Write((byte)(value >> 48));
Write((byte)(value >> 40));
@ -239,6 +295,7 @@ namespace System.Buffers
Write((byte)(value >> 16));
Write((byte)(value >> 8));
Write((byte)value);
}
else
{
@ -265,7 +322,11 @@ namespace System.Buffers
{
if (!BinaryPrimitives.TryWriteUInt64BigEndian(_first.Slice(Position), value))
{
// Not enough space. Split the spans
if (BitConverter.IsLittleEndian)
{
value = BinaryPrimitives.ReverseEndianness(value);
}
Write((byte)(value >> 56));
Write((byte)(value >> 48));
Write((byte)(value >> 40));

View file

@ -25,53 +25,60 @@ namespace Server
{
public class BufferWriter : IGenericWriter
{
private const int LargeByteBufferSize = 256;
private readonly Encoding m_Encoding;
private readonly bool m_PrefixStrings;
protected long Index { get; set; }
private byte[] m_CharacterBuffer;
private int m_MaxBufferChars;
private byte[] _buffer;
public BufferWriter(byte[] buffer, bool prefixStr)
{
m_PrefixStrings = prefixStr;
m_Encoding = Utility.UTF8;
Buffer = buffer;
_buffer = buffer;
}
public BufferWriter(bool prefixStr)
{
m_PrefixStrings = prefixStr;
m_Encoding = Utility.UTF8;
Buffer = new byte[BufferSize];
_buffer = new byte[BufferSize];
}
public virtual long Position => Index;
protected virtual int BufferSize => 256;
public byte[] Buffer { get; protected set; }
public byte[] Buffer => _buffer;
public virtual void Close()
{
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Resize(int size)
{
var copy = new byte[size];
System.Buffer.BlockCopy(Buffer, 0, copy, 0, Math.Min(size, Buffer.Length));
Buffer = copy;
// We shouldn't ever resize to a 0 length buffer. That is dangerous
if (size <= 0)
{
size = BufferSize;
}
Array.Resize(ref _buffer, size);
}
public virtual void Flush()
{
Resize(Buffer.Length * 2);
Resize(_buffer.Length * 2);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void FlushIfNeeded(int amount)
{
if (Index + amount > _buffer.Length)
{
Flush();
}
}
public void Reset()
@ -79,6 +86,23 @@ namespace Server
Index = 0;
}
public void Write(ReadOnlySpan<byte> bytes)
{
var remaining = bytes.Length;
var idx = 0;
while (remaining > 0)
{
FlushIfNeeded(remaining);
var count = Math.Min((int)(_buffer.Length - Index), remaining);
bytes.Slice(idx, count).CopyTo(_buffer.AsSpan((int)Index, count));
idx += count;
Index += count;
remaining -= count;
}
}
public virtual long Seek(long offset, SeekOrigin origin)
{
return origin switch
@ -96,21 +120,11 @@ namespace Server
while (v >= 0x80)
{
if (Index + 1 > Buffer.Length)
{
Flush();
}
Buffer[Index++] = (byte)(v | 0x80);
Write((byte)(v | 0x80));
v >>= 7;
}
if (Index + 1 > Buffer.Length)
{
Flush();
}
Buffer[Index++] = (byte)v;
Write((byte)v);
}
public void Write(string value)
@ -119,22 +133,11 @@ namespace Server
{
if (value == null)
{
if (Index + 1 > Buffer.Length)
{
Flush();
}
Buffer[Index++] = 0;
Write((byte)0);
}
else
{
if (Index + 1 > Buffer.Length)
{
Flush();
}
Buffer[Index++] = 1;
Write((byte)1);
InternalWriteString(value);
}
}
@ -176,7 +179,10 @@ namespace Server
public void Write(IPAddress value)
{
Write(Utility.GetLongAddressValue(value));
Span<byte> stack = stackalloc byte[16];
value.TryWriteBytes(stack, out var bytesWritten);
Write((byte)bytesWritten);
Write(stack.Slice(0, bytesWritten));
}
public void Write(TimeSpan value)
@ -196,100 +202,79 @@ namespace Server
public void Write(long value)
{
if (Index + 8 > Buffer.Length)
{
Flush();
}
FlushIfNeeded(8);
Buffer[Index] = (byte)value;
Buffer[Index + 1] = (byte)(value >> 8);
Buffer[Index + 2] = (byte)(value >> 16);
Buffer[Index + 3] = (byte)(value >> 24);
Buffer[Index + 4] = (byte)(value >> 32);
Buffer[Index + 5] = (byte)(value >> 40);
Buffer[Index + 6] = (byte)(value >> 48);
Buffer[Index + 7] = (byte)(value >> 56);
_buffer[Index] = (byte)value;
_buffer[Index + 1] = (byte)(value >> 8);
_buffer[Index + 2] = (byte)(value >> 16);
_buffer[Index + 3] = (byte)(value >> 24);
_buffer[Index + 4] = (byte)(value >> 32);
_buffer[Index + 5] = (byte)(value >> 40);
_buffer[Index + 6] = (byte)(value >> 48);
_buffer[Index + 7] = (byte)(value >> 56);
Index += 8;
}
public void Write(ulong value)
{
if (Index + 8 > Buffer.Length)
{
Flush();
}
FlushIfNeeded(8);
Buffer[Index] = (byte)value;
Buffer[Index + 1] = (byte)(value >> 8);
Buffer[Index + 2] = (byte)(value >> 16);
Buffer[Index + 3] = (byte)(value >> 24);
Buffer[Index + 4] = (byte)(value >> 32);
Buffer[Index + 5] = (byte)(value >> 40);
Buffer[Index + 6] = (byte)(value >> 48);
Buffer[Index + 7] = (byte)(value >> 56);
_buffer[Index] = (byte)value;
_buffer[Index + 1] = (byte)(value >> 8);
_buffer[Index + 2] = (byte)(value >> 16);
_buffer[Index + 3] = (byte)(value >> 24);
_buffer[Index + 4] = (byte)(value >> 32);
_buffer[Index + 5] = (byte)(value >> 40);
_buffer[Index + 6] = (byte)(value >> 48);
_buffer[Index + 7] = (byte)(value >> 56);
Index += 8;
}
public void Write(int value)
{
if (Index + 4 > Buffer.Length)
{
Flush();
}
FlushIfNeeded(4);
Buffer[Index] = (byte)value;
Buffer[Index + 1] = (byte)(value >> 8);
Buffer[Index + 2] = (byte)(value >> 16);
Buffer[Index + 3] = (byte)(value >> 24);
_buffer[Index] = (byte)value;
_buffer[Index + 1] = (byte)(value >> 8);
_buffer[Index + 2] = (byte)(value >> 16);
_buffer[Index + 3] = (byte)(value >> 24);
Index += 4;
}
public void Write(uint value)
{
if (Index + 4 > Buffer.Length)
{
Flush();
}
FlushIfNeeded(4);
Buffer[Index] = (byte)value;
Buffer[Index + 1] = (byte)(value >> 8);
Buffer[Index + 2] = (byte)(value >> 16);
Buffer[Index + 3] = (byte)(value >> 24);
_buffer[Index] = (byte)value;
_buffer[Index + 1] = (byte)(value >> 8);
_buffer[Index + 2] = (byte)(value >> 16);
_buffer[Index + 3] = (byte)(value >> 24);
Index += 4;
}
public void Write(short value)
{
if (Index + 2 > Buffer.Length)
{
Flush();
}
FlushIfNeeded(2);
Buffer[Index] = (byte)value;
Buffer[Index + 1] = (byte)(value >> 8);
_buffer[Index] = (byte)value;
_buffer[Index + 1] = (byte)(value >> 8);
Index += 2;
}
public void Write(ushort value)
{
if (Index + 2 > Buffer.Length)
{
Flush();
}
FlushIfNeeded(2);
Buffer[Index] = (byte)value;
Buffer[Index + 1] = (byte)(value >> 8);
_buffer[Index] = (byte)value;
_buffer[Index + 1] = (byte)(value >> 8);
Index += 2;
}
public unsafe void Write(double value)
{
if (Index + 8 > Buffer.Length)
{
Flush();
}
FlushIfNeeded(8);
fixed (byte* pBuffer = Buffer)
fixed (byte* pBuffer = _buffer)
{
*(double*)(pBuffer + Index) = value;
}
@ -299,12 +284,9 @@ namespace Server
public unsafe void Write(float value)
{
if (Index + 4 > Buffer.Length)
{
Flush();
}
FlushIfNeeded(4);
fixed (byte* pBuffer = Buffer)
fixed (byte* pBuffer = _buffer)
{
*(float*)(pBuffer + Index) = value;
}
@ -312,56 +294,23 @@ namespace Server
Index += 4;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Write(byte value)
{
if (Index + 1 > Buffer.Length)
{
Flush();
}
Buffer[Index++] = value;
}
public void Write(byte[] value, int length)
{
var remaining = length;
var idx = 0;
while (remaining > 0)
{
int size = Math.Min(Buffer.Length - (int)Index, remaining);
System.Buffer.BlockCopy(value, idx, Buffer, (int)Index, size);
// value.Slice(idx).CopyTo(m_Buffer.AsSpan(m_Index, size));
remaining -= size;
Index += size;
idx += size;
if (Index == Buffer.Length)
{
Flush();
}
}
FlushIfNeeded(1);
_buffer[Index++] = value;
}
public void Write(sbyte value)
{
if (Index + 1 > Buffer.Length)
{
Flush();
}
Buffer[Index++] = (byte)value;
FlushIfNeeded(1);
_buffer[Index++] = (byte)value;
}
public void Write(bool value)
{
if (Index + 1 > Buffer.Length)
{
Flush();
}
Buffer[Index++] = (byte)(value ? 1 : 0);
FlushIfNeeded(1);
_buffer[Index++] = value ? 1 : 0;
}
public void Write(Point3D value)
@ -391,26 +340,12 @@ namespace Server
public void Write(Map value)
{
if (value != null)
{
Write((byte)value.MapIndex);
}
else
{
Write((byte)0xFF);
}
Write((byte)(value?.MapIndex ?? 0xFF));
}
public void Write(Race value)
{
if (value != null)
{
Write((byte)value.RaceIndex);
}
else
{
Write((byte)0xFF);
}
Write((byte)(value?.RaceIndex ?? 0xFF));
}
public void WriteEntity(IEntity value)
@ -430,14 +365,7 @@ namespace Server
public void Write(BaseGuild value)
{
if (value == null)
{
Write(0);
}
else
{
Write(value.Serial);
}
Write(value?.Serial ?? 0);
}
public void WriteItem<T>(T value) where T : Item
@ -737,49 +665,29 @@ namespace Server
internal void InternalWriteString(string value)
{
var length = m_Encoding.GetByteCount(value);
var remaining = m_Encoding.GetByteCount(value);
WriteEncodedInt(length);
if (m_CharacterBuffer == null)
WriteEncodedInt(remaining);
if (remaining == 0)
{
m_CharacterBuffer = new byte[LargeByteBufferSize];
m_MaxBufferChars = LargeByteBufferSize / m_Encoding.GetMaxByteCount(1);
return;
}
if (length > LargeByteBufferSize)
// It is much faster to encode to stack buffer, then copy to the real buffer
Span<byte> span = stackalloc byte[Math.Min(BufferSize, 256)];
var maxChars = span.Length / m_Encoding.GetMaxByteCount(1);
var charsLeft = value.Length;
var current = 0;
while (charsLeft > 0)
{
var current = 0;
var charsLeft = value.Length;
var charCount = Math.Min(charsLeft, maxChars);
var bytesWritten = m_Encoding.GetBytes(value.AsSpan(current, charCount), span);
remaining -= bytesWritten;
charsLeft -= charCount;
current += charCount;
while (charsLeft > 0)
{
var charCount = Math.Min(charsLeft, m_MaxBufferChars);
var byteLength = m_Encoding.GetBytes(value, current, charCount, m_CharacterBuffer, 0);
if (Index + byteLength > Buffer.Length)
{
Flush();
}
System.Buffer.BlockCopy(m_CharacterBuffer, 0, Buffer, (int)Index, byteLength);
Index += byteLength;
current += charCount;
charsLeft -= charCount;
}
}
else
{
var byteLength = m_Encoding.GetBytes(value, 0, value.Length, m_CharacterBuffer, 0);
if (Index + byteLength > Buffer.Length)
{
Flush();
}
System.Buffer.BlockCopy(m_CharacterBuffer, 0, Buffer, (int)Index, byteLength);
Index += byteLength;
Write(span.Slice(0, bytesWritten));
}
}
}

View file

@ -39,7 +39,6 @@ namespace Server
void Write(double value);
void Write(float value);
void Write(byte value);
void Write(byte[] value, int length);
void Write(sbyte value);
void Write(bool value);
void WriteEncodedInt(int value);
@ -82,6 +81,7 @@ namespace Server
void Write(HashSet<BaseGuild> list, bool tidy);
void WriteGuildSet<T>(HashSet<T> set) where T : BaseGuild;
void WriteGuildSet<T>(HashSet<T> set, bool tidy) where T : BaseGuild;
void Write(ReadOnlySpan<byte> bytes);
long Seek(long offset, SeekOrigin origin);
}

View file

@ -528,15 +528,15 @@ namespace Server
var bin = new BinaryFileWriter(binPath, true);
idx.Write(entities.Count);
foreach (var mob in entities.Values)
foreach (var e in entities.Values)
{
long start = bin.Position;
idx.Write(mob.TypeRef);
idx.Write(mob.Serial);
idx.Write(e.TypeRef);
idx.Write(e.Serial);
idx.Write(start);
mob.SerializeTo(bin);
e.SerializeTo(bin);
idx.Write((int)(bin.Position - start));
}
@ -775,7 +775,7 @@ namespace Server
public static void SerializeTo(this ISerializable entity, IGenericWriter writer)
{
var saveBuffer = entity.SaveBuffer;
writer.Write(saveBuffer.Buffer, (int)saveBuffer.Position);
writer.Write(saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position));
// Resize to exact buffer size
entity.SaveBuffer.Resize((int)entity.SaveBuffer.Position);

View file

@ -735,7 +735,7 @@ namespace Server.Mobiles
public bool Contains(Point3D p) => Region.Contains(p);
public static void LoadTable()
public static void Initialize()
{
ICollection list = Map.Felucca.Regions.Values;
@ -757,11 +757,6 @@ namespace Server.Mobiles
public static EDI Find(string name)
{
if (m_Table == null)
{
LoadTable();
}
if (name == null || m_Table == null)
{
return null;