fix(core): Updates to serialization (#590)

- [X] Adds `AdhocPersistence` which replaces RunUO `Persistence`
- [X] Fixes a few minor bugs with EntityPersistence deserialization
- [X] Reverts/updates some serialization changes

Projects/Server/Serialization/AdhocPersistence.cs

@@ -0,0 +1,59 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2021 - ModernUO Development Team                       *
+ * Email: hi@modernuo.com                                                *
+ * File: AdhocPersistence.cs                                             *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+using System;
+using System.IO;
+
+namespace Server
+{
+    public static class AdhocPersistence
+    {
+        public static void Serialize(string filePath, Action<IGenericWriter> serializer)
+        {
+            var fullPath = Path.Combine(Core.BaseDirectory, filePath);
+            var file = new FileInfo(fullPath);
+            file.Directory?.Create();
+
+            using var bin = new BinaryFileWriter(fullPath, true);
+            serializer(bin);
+        }
+
+        public static void Deserialize(string filePath, Action<IGenericReader> deserializer)
+        {
+            var fullPath = Path.Combine(Core.BaseDirectory, filePath);
+            var file = new FileInfo(fullPath);
+            file.Directory?.Create();
+
+            if (!file.Exists)
+            {
+                return;
+            }
+
+            try
+            {
+                using FileStream fs = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
+                using var br = new BinaryFileReader(fs);
+                deserializer(br);
+            }
+            catch (Exception e)
+            {
+                Utility.PushColor(ConsoleColor.Red);
+                Console.WriteLine($"***** Bad deserialize of {file.FullName} *****");
+                Console.WriteLine(e.ToString());
+                Utility.PopColor();
+            }
+        }
+    }
+}

Projects/Server/Serialization/BinaryFileReader.cs

@@ -0,0 +1,111 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2021 - ModernUO Development Team                       *
+ * Email: hi@modernuo.com                                                *
+ * File: BinaryFileReader.cs                                             *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+using System;
+using System.IO;
+using System.Net;
+
+namespace Server
+{
+    public class BinaryFileReader : IGenericReader, IDisposable
+    {
+        private readonly BinaryReader _reader;
+
+        public BinaryFileReader(BinaryReader br) => _reader = br;
+
+        public BinaryFileReader(Stream stream) => _reader = new BinaryReader(stream);
+
+        public void Close() => _reader.Close();
+
+        public string ReadString(bool intern = false)
+        {
+            var str = _reader.ReadString();
+            return intern ? Utility.Intern(str) : str;
+        }
+
+        public DateTime ReadDateTime() => new(_reader.ReadInt64(), DateTimeKind.Utc);
+
+        public TimeSpan ReadTimeSpan() => new(_reader.ReadInt64());
+
+        public DateTime ReadDeltaTime() => new(_reader.ReadInt64() + DateTime.UtcNow.Ticks, DateTimeKind.Utc);
+
+        public decimal ReadDecimal() => _reader.ReadDecimal();
+
+        public long ReadLong() => _reader.ReadInt64();
+
+        public ulong ReadULong() => _reader.ReadUInt64();
+
+        public int ReadInt() => _reader.ReadInt32();
+
+        public uint ReadUInt() => _reader.ReadUInt32();
+
+        public short ReadShort() => _reader.ReadInt16();
+
+        public ushort ReadUShort() => _reader.ReadUInt16();
+
+        public double ReadDouble() => _reader.ReadDouble();
+
+        public float ReadFloat() => _reader.ReadSingle();
+
+        public byte ReadByte() => _reader.ReadByte();
+
+        public sbyte ReadSByte() => _reader.ReadSByte();
+
+        public bool ReadBool() => _reader.ReadBoolean();
+
+        public int ReadEncodedInt()
+        {
+            int v = 0, shift = 0;
+            byte b;
+
+            do
+            {
+                b = _reader.ReadByte();
+                v |= (b & 0x7F) << shift;
+                shift += 7;
+            }
+            while (b >= 0x80);
+
+            return v;
+        }
+
+        public IPAddress ReadIPAddress()
+        {
+            byte length = ReadByte();
+            // Either 2 ushorts, or 8 ushorts
+            Span<byte> integer = stackalloc byte[length];
+            Read(integer);
+            return Utility.Intern(new IPAddress(integer));
+        }
+
+        public Point3D ReadPoint3D() => new(ReadInt(), ReadInt(), ReadInt());
+
+        public Point2D ReadPoint2D() => new(ReadInt(), ReadInt());
+
+        public Rectangle2D ReadRect2D() => new(ReadPoint2D(), ReadPoint2D());
+
+        public Rectangle3D ReadRect3D() => new(ReadPoint3D(), ReadPoint3D());
+
+        public Map ReadMap() => Map.Maps[ReadByte()];
+
+        public Race ReadRace() => Race.Races[ReadByte()];
+
+        public int Read(Span<byte> buffer) => _reader.Read(buffer);
+
+        public long Seek(long offset, SeekOrigin origin) => _reader.BaseStream.Seek(offset, origin);
+
+        public void Dispose() => Close();
+    }
+}

Projects/Server/Serialization/BinaryFileWriter.cs

@@ -66,9 +66,6 @@ namespace Server
             return _position = _file.Seek(offset, origin);
         }
 
-        public void Dispose()
-        {
-            Close();
-        }
+        public void Dispose() => Close();
     }
 }

Projects/Server/Serialization/BufferReader.cs

@@ -15,14 +15,10 @@
 
 using System;
 using System.Buffers.Binary;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Net;
-using System.Runtime.CompilerServices;
 using System.Text;
-using Server.Guilds;
-using Server.Network;
 using Server.Text;
 
 namespace Server
@@ -33,25 +29,23 @@ namespace Server
         private byte[] _buffer;
         private int _position;
 
-        public int Position => _position;
+        public long Position => _position;
 
-        public byte[] Buffer => _buffer;
-
-        public BufferReader(byte[] buffer)
+        public BufferReader(byte[] buffer, Encoding encoding = null)
         {
             _buffer = buffer;
-            _encoding = TextEncoding.UTF8;
+            _encoding = encoding ?? TextEncoding.UTF8;
         }
 
-        public void SwapBuffers(byte[] newBuffer, out byte[] oldBuffer)
+        public void Reset(byte[] newBuffer, out byte[] oldBuffer)
         {
             oldBuffer = _buffer;
             _buffer = newBuffer;
             _position = 0;
         }
 
-        [Obsolete("RunUO backward compatible method that should be replaced.")]
-        public string ReadString()
+        // Compatible with BinaryReader.ReadString()
+        public string ReadString(bool intern = false)
         {
             if (!ReadBool())
             {
@@ -59,146 +53,76 @@ namespace Server
             }
 
             var length = ReadEncodedInt();
-            return length <= 0 ? "" : ReadString(_encoding, false, length);
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadString(Encoding encoding, bool safeString = false, int fixedLength = -1)
-        {
-            int sizeT = TextEncoding.GetByteLengthForEncoding(encoding);
-
-            bool isFixedLength = fixedLength > -1;
-
-            var remaining = _buffer.Length - _position;
-            int size;
-
-            if (isFixedLength)
+            if (length <= 0)
             {
-                size = fixedLength * sizeT;
-                if (size > remaining)
-                {
-                    throw new OutOfMemoryException();
-                }
-            }
-            else
-            {
-                size = remaining - (remaining & (sizeT - 1));
+                return intern ? Utility.Intern("") : "";
             }
 
-            var buffer = _buffer.AsSpan(Position, size);
-
-            int index = buffer.IndexOfTerminator(sizeT);
-
-            var span = buffer[..(index < 0 ? size : index)];
-            _position += isFixedLength || index < 0 ? size : index + sizeT;
-            return TextEncoding.GetString(span, encoding, safeString);
+            var str = TextEncoding.GetString(_buffer.AsSpan(_position, length), _encoding);
+            _position += length;
+            return intern ? Utility.Intern(str) : str;
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadLittleUniSafe(int fixedLength) => ReadString(TextEncoding.UnicodeLE, true, fixedLength);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadLittleUniSafe() => ReadString(TextEncoding.UnicodeLE, true);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadLittleUni(int fixedLength) => ReadString(TextEncoding.UnicodeLE, false, fixedLength);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadLittleUni() => ReadString(TextEncoding.UnicodeLE);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadBigUniSafe(int fixedLength) => ReadString(TextEncoding.Unicode, true, fixedLength);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadBigUniSafe() => ReadString(TextEncoding.Unicode, true);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadBigUni(int fixedLength) => ReadString(TextEncoding.Unicode, false, fixedLength);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadBigUni() => ReadString(TextEncoding.Unicode);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadUTF8Safe(int fixedLength) => ReadString(TextEncoding.UTF8, true, fixedLength);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadUTF8Safe() => ReadString(TextEncoding.UTF8, true);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadUTF8() => ReadString(TextEncoding.UTF8);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadAsciiSafe(int fixedLength) => ReadString(Encoding.ASCII, true, fixedLength);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadAsciiSafe() => ReadString(Encoding.ASCII, true);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadAscii(int fixedLength) => ReadString(Encoding.ASCII, false, fixedLength);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public string ReadAscii() => ReadString(Encoding.ASCII);
-
         public DateTime ReadDateTime() => new(ReadLong(), DateTimeKind.Utc);
 
         public TimeSpan ReadTimeSpan() => new(ReadLong());
 
         public DateTime ReadDeltaTime() => new(ReadLong() + DateTime.UtcNow.Ticks, DateTimeKind.Utc);
 
-        public decimal ReadDecimal() => new(new[] { ReadInt(), ReadInt(), ReadInt(), ReadInt() });
+        public decimal ReadDecimal() => new(stackalloc int[4] { ReadInt(), ReadInt(), ReadInt(), ReadInt() });
 
         public long ReadLong()
         {
-            var v = BinaryPrimitives.ReadInt64LittleEndian(_buffer.AsSpan(Position, 8));
+            var v = BinaryPrimitives.ReadInt64LittleEndian(_buffer.AsSpan(_position, 8));
             _position += 8;
             return v;
         }
 
         public ulong ReadULong()
         {
-            var v = BinaryPrimitives.ReadUInt64LittleEndian(_buffer.AsSpan(Position, 8));
+            var v = BinaryPrimitives.ReadUInt64LittleEndian(_buffer.AsSpan(_position, 8));
             _position += 8;
             return v;
         }
 
         public int ReadInt()
         {
-            var v = BinaryPrimitives.ReadInt32LittleEndian(_buffer.AsSpan(Position, 4));
+            var v = BinaryPrimitives.ReadInt32LittleEndian(_buffer.AsSpan(_position, 4));
             _position += 4;
             return v;
         }
 
         public uint ReadUInt()
         {
-            var v = BinaryPrimitives.ReadUInt32LittleEndian(_buffer.AsSpan(Position, 4));
+            var v = BinaryPrimitives.ReadUInt32LittleEndian(_buffer.AsSpan(_position, 4));
             _position += 4;
             return v;
         }
 
         public short ReadShort()
         {
-            var v = BinaryPrimitives.ReadInt16LittleEndian(_buffer.AsSpan(Position, 2));
+            var v = BinaryPrimitives.ReadInt16LittleEndian(_buffer.AsSpan(_position, 2));
             _position += 2;
             return v;
         }
 
         public ushort ReadUShort()
         {
-            var v = BinaryPrimitives.ReadUInt16LittleEndian(_buffer.AsSpan(Position, 2));
+            var v = BinaryPrimitives.ReadUInt16LittleEndian(_buffer.AsSpan(_position, 2));
             _position += 2;
             return v;
         }
 
         public double ReadDouble()
         {
-            var v = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.AsSpan(Position, 8));
+            var v = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.AsSpan(_position, 8));
             _position += 8;
             return v;
         }
 
         public float ReadFloat()
         {
-            var v = BinaryPrimitives.ReadSingleLittleEndian(_buffer.AsSpan(Position, 4));
+            var v = BinaryPrimitives.ReadSingleLittleEndian(_buffer.AsSpan(_position, 4));
             _position += 4;
             return v;
         }
@@ -230,7 +154,7 @@ namespace Server
             // Either 2 ushorts, or 8 ushorts
             Span<byte> integer = stackalloc byte[length];
             Read(integer);
-            return new IPAddress(integer);
+            return Utility.Intern(new IPAddress(integer));
         }
 
         public Point3D ReadPoint3D() => new(ReadInt(), ReadInt(), ReadInt());
@@ -243,71 +167,22 @@ namespace Server
 
         public Map ReadMap() => Map.Maps[ReadByte()];
 
-        public T ReadEntity<T>() where T : class, ISerializable
-        {
-            Serial serial = ReadUInt();
-
-            // Special case for now:
-            if (typeof(T).IsAssignableTo(typeof(BaseGuild)))
-            {
-                return World.FindGuild(serial) as T;
-            }
-
-            return World.FindEntity(serial) as T;
-        }
-
-        public List<T> ReadEntityList<T>() where T : class, ISerializable
-        {
-            var count = ReadInt();
-
-            var list = new List<T>(count);
-
-            for (var i = 0; i < count; ++i)
-            {
-                var entity = ReadEntity<T>();
-                if (entity != null)
-                {
-                    list.Add(entity);
-                }
-            }
-
-            return list;
-        }
-
-        public HashSet<T> ReadEntitySet<T>() where T : class, ISerializable
-        {
-            var count = ReadInt();
-
-            var set = new HashSet<T>(count);
-
-            for (var i = 0; i < count; ++i)
-            {
-                var entity = ReadEntity<T>();
-                if (entity != null)
-                {
-                    set.Add(entity);
-                }
-            }
-
-            return set;
-        }
-
         public Race ReadRace() => Race.Races[ReadByte()];
 
         public int Read(Span<byte> buffer)
         {
             var length = buffer.Length;
-            if (length > _buffer.Length - Position)
+            if (length > _buffer.Length - _position)
             {
                 throw new OutOfMemoryException();
             }
 
-            _buffer.AsSpan(Position, length).CopyTo(buffer);
+            _buffer.AsSpan(_position, length).CopyTo(buffer);
             _position += length;
             return length;
         }
 
-        public virtual int Seek(int offset, SeekOrigin origin)
+        public virtual long Seek(long offset, SeekOrigin origin)
         {
             Debug.Assert(
                 origin != SeekOrigin.End || offset <= 0 && offset > -_buffer.Length,
@@ -322,12 +197,19 @@ namespace Server
                 "Attempting to seek to an invalid position using SeekOrigin.Current"
             );
 
-            return _position = Math.Max(0, origin switch
+            var position = Math.Max(0L, origin switch
             {
                 SeekOrigin.Current => _position + offset,
                 SeekOrigin.End     => _buffer.Length + offset,
                 _                  => offset // Begin
             });
+
+            if (position > int.MaxValue)
+            {
+                throw new ArgumentException($"BufferReader does not support {nameof(offset)} beyond Int32.MaxValue");
+            }
+
+            return _position;
         }
     }
 }

Projects/Server/Serialization/BufferWriter.cs

@@ -14,13 +14,11 @@
  *************************************************************************/
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Net;
 using System.Runtime.CompilerServices;
 using System.Text;
-using Server.Network;
 using Server.Text;
 
 namespace Server
@@ -387,35 +385,6 @@ namespace Server
             Write((byte)(value?.RaceIndex ?? 0xFF));
         }
 
-        public void Write(ISerializable value)
-        {
-            Write(value?.Deleted != false ? Serial.MinusOne : value.Serial);
-        }
-
-        public void Write<T>(ICollection<T> coll) where T : class, ISerializable
-        {
-            Write(coll.Count);
-            foreach (var entry in coll)
-            {
-                Write(entry);
-            }
-        }
-
-        public void Write<T>(ICollection<T> coll, Action<IGenericWriter, T> action) where T : class, ISerializable
-        {
-            if (coll == null)
-            {
-                Write(0);
-                return;
-            }
-
-            Write(coll.Count);
-            foreach (var entry in coll)
-            {
-                action(this, entry);
-            }
-        }
-
         internal void InternalWriteString(string value)
         {
             var remaining = m_Encoding.GetByteCount(value);

Projects/Server/Serialization/GenericPersistence.cs

@@ -53,7 +53,6 @@ namespace Server
             void Deserialize(string savePath)
             {
                 var path = Path.Combine(savePath, name);
-
                 AssemblyHandler.EnsureDirectory(path);
 
                 string binPath = Path.Combine(path, $"{name}.bin");
@@ -65,10 +64,9 @@ namespace Server
 
                 try
                 {
-                    using FileStream bin = new FileStream(binPath, FileMode.Open, FileAccess.Read, FileShare.Read);
-                    var buffer = GC.AllocateUninitializedArray<byte>((int)bin.Length);
-                    bin.Read(buffer);
-                    deserializer(new BufferReader(buffer));
+                    using FileStream fs = new FileStream(binPath, FileMode.Open, FileAccess.Read, FileShare.Read);
+                    using var br = new BinaryFileReader(fs);
+                    deserializer(br);
                 }
                 catch (Exception e)
                 {

Projects/Server/Serialization/IGenericReader.cs

@@ -14,14 +14,14 @@
  *************************************************************************/
 
 using System;
-using System.Collections.Generic;
+using System.IO;
 using System.Net;
 
 namespace Server
 {
     public interface IGenericReader
     {
-        string ReadString();
+        string ReadString(bool intern = false);
         DateTime ReadDateTime();
         TimeSpan ReadTimeSpan();
         DateTime ReadDeltaTime();
@@ -44,10 +44,8 @@ namespace Server
         Rectangle2D ReadRect2D();
         Rectangle3D ReadRect3D();
         Map ReadMap();
-        T ReadEntity<T>() where T : class, ISerializable;
-        List<T> ReadEntityList<T>() where T : class, ISerializable;
-        HashSet<T> ReadEntitySet<T>() where T : class, ISerializable;
         Race ReadRace();
         int Read(Span<byte> buffer);
+        long Seek(long offset, SeekOrigin origin);
     }
 }

Projects/Server/Serialization/IGenericWriter.cs

@@ -14,7 +14,6 @@
  *************************************************************************/
 
 using System;
-using System.Collections.Generic;
 using System.IO;
 using System.Net;
 
@@ -47,9 +46,7 @@ namespace Server
         void Write(Rectangle2D value);
         void Write(Rectangle3D value);
         void Write(Map value);
-        void Write(ISerializable value);
         void Write(Race value);
-        void Write<T>(ICollection<T> list) where T : class, ISerializable;
         void Write(ReadOnlySpan<byte> bytes);
 
         long Seek(long offset, SeekOrigin origin);