Speeds up world loading (#315)

- [X] Speeds up world loading
- [X] Removes PeekChar cause it wasn't used
- [X] Adds a BufferReader
- [X] Removes BinaryFileReader
- [X] Removes reading/writing `char` since it is not consistent and will cause a deserialize issue if used.

Bumps version release

Projects/Server/Geometry/Rectangle2D.cs

@@ -29,6 +29,12 @@ namespace Server
             m_End = new Point2D(end);
         }
 
+        public Rectangle2D(Point2D start, Point2D end)
+        {
+            m_Start = start;
+            m_End = end;
+        }
+
         public Rectangle2D(int x, int y, int width, int height)
         {
             m_Start = new Point2D(x, y);

Projects/Server/Serialization/BufferReader.cs

@@ -2,7 +2,7 @@
  * ModernUO                                                              *
  * Copyright 2019-2020 - ModernUO Development Team                       *
  * Email: hi@modernuo.com                                                *
- * File: BinaryFileReader.cs                                             *
+ * File: BufferReader.cs                                                 *
  *                                                                       *
  * This program is free software: you can redistribute it and/or modify  *
  * it under the terms of the GNU General Public License as published by  *
@@ -14,26 +14,48 @@
  *************************************************************************/
 
 using System;
+using System.Buffers.Binary;
 using System.Collections.Generic;
-using System.IO;
 using System.Net;
+using System.Text;
 using Server.Guilds;
 
 namespace Server
 {
-    public sealed class BinaryFileReader : IGenericReader
+    public class BufferReader : IGenericReader
     {
-        private readonly BinaryReader m_File;
+        private readonly Encoding _encoding;
+        private ArraySegment<byte> _segment;
+        public int Position { get; private set; }
 
-        public BinaryFileReader(BinaryReader br) => m_File = br;
+        public BufferReader(ArraySegment<byte> segment)
+        {
+            _segment = segment;
+            _encoding = Utility.UTF8;
+        }
 
-        public long Position => m_File.BaseStream.Position;
+        public string ReadString()
+        {
+            if (!ReadBool())
+            {
+                return null;
+            }
 
-        public string ReadString() => ReadByte() != 0 ? m_File.ReadString() : null;
+            var length = ReadEncodedInt();
+            var s = length == 0 ? "" : _encoding.GetString(_segment.AsSpan(Position, length));
+            Position += length;
+            return s;
+        }
+
+        public DateTime ReadDateTime() => new DateTime(ReadLong());
+
+        public DateTimeOffset ReadDateTimeOffset() => new DateTimeOffset(ReadLong(), ReadTimeSpan());
+
+        public TimeSpan ReadTimeSpan() => new TimeSpan(ReadLong());
 
         public DateTime ReadDeltaTime()
         {
-            var ticks = m_File.ReadInt64();
+            var ticks = ReadLong();
             var now = DateTime.UtcNow.Ticks;
 
             if (ticks > 0 && ticks + now < 0)
@@ -56,7 +78,69 @@ namespace Server
             }
         }
 
-        public IPAddress ReadIPAddress() => new IPAddress(m_File.ReadInt64());
+        public decimal ReadDecimal() => new decimal(new[] { ReadInt(), ReadInt(), ReadInt(), ReadInt() });
+
+        public long ReadLong()
+        {
+            var v = BinaryPrimitives.ReadInt64LittleEndian(_segment.AsSpan(Position, 8));
+            Position += 8;
+            return v;
+        }
+
+        public ulong ReadULong()
+        {
+            var v = BinaryPrimitives.ReadUInt64LittleEndian(_segment.AsSpan(Position, 8));
+            Position += 8;
+            return v;
+        }
+
+        public int ReadInt()
+        {
+            var v = BinaryPrimitives.ReadInt32LittleEndian(_segment.AsSpan(Position, 4));
+            Position += 4;
+            return v;
+        }
+
+        public uint ReadUInt()
+        {
+            var v = BinaryPrimitives.ReadUInt32LittleEndian(_segment.AsSpan(Position, 4));
+            Position += 4;
+            return v;
+        }
+
+        public short ReadShort()
+        {
+            var v = BinaryPrimitives.ReadInt16LittleEndian(_segment.AsSpan(Position, 2));
+            Position += 2;
+            return v;
+        }
+
+        public ushort ReadUShort()
+        {
+            var v = BinaryPrimitives.ReadUInt16LittleEndian(_segment.AsSpan(Position, 2));
+            Position += 2;
+            return v;
+        }
+
+        public double ReadDouble()
+        {
+            var v = BinaryPrimitives.ReadDoubleLittleEndian(_segment.AsSpan(Position, 8));
+            Position += 8;
+            return v;
+        }
+
+        public float ReadFloat()
+        {
+            var v = BinaryPrimitives.ReadSingleLittleEndian(_segment.AsSpan(Position, 4));
+            Position += 4;
+            return v;
+        }
+
+        public byte ReadByte() => _segment[Position++];
+
+        public sbyte ReadSByte() => (sbyte)_segment[Position++];
+
+        public bool ReadBool() => _segment[Position++] != 0;
 
         public int ReadEncodedInt()
         {
@@ -65,7 +149,7 @@ namespace Server
 
             do
             {
-                b = m_File.ReadByte();
+                b = ReadByte();
                 v |= (b & 0x7F) << shift;
                 shift += 7;
             } while (b >= 0x80);
@@ -73,43 +157,7 @@ namespace Server
             return v;
         }
 
-        public DateTime ReadDateTime() => new DateTime(m_File.ReadInt64());
+        public IPAddress ReadIPAddress() => new IPAddress(ReadLong());
-
-        public DateTimeOffset ReadDateTimeOffset()
-        {
-            var ticks = m_File.ReadInt64();
-            var offset = new TimeSpan(m_File.ReadInt64());
-
-            return new DateTimeOffset(ticks, offset);
-        }
-
-        public TimeSpan ReadTimeSpan() => new TimeSpan(m_File.ReadInt64());
-
-        public decimal ReadDecimal() => m_File.ReadDecimal();
-
-        public long ReadLong() => m_File.ReadInt64();
-
-        public ulong ReadULong() => m_File.ReadUInt64();
-
-        public int ReadInt() => m_File.ReadInt32();
-
-        public uint ReadUInt() => m_File.ReadUInt32();
-
-        public short ReadShort() => m_File.ReadInt16();
-
-        public ushort ReadUShort() => m_File.ReadUInt16();
-
-        public double ReadDouble() => m_File.ReadDouble();
-
-        public float ReadFloat() => m_File.ReadSingle();
-
-        public char ReadChar() => m_File.ReadChar();
-
-        public byte ReadByte() => m_File.ReadByte();
-
-        public sbyte ReadSByte() => m_File.ReadSByte();
-
-        public bool ReadBool() => m_File.ReadBoolean();
 
         public Point3D ReadPoint3D() => new Point3D(ReadInt(), ReadInt(), ReadInt());
 
@@ -259,15 +307,17 @@ namespace Server
 
         public Race ReadRace() => Race.Races[ReadByte()];
 
-        public bool End() => m_File.PeekChar() == -1;
+        public int Read(Span<byte> buffer)
-
-        public int Read(Span<byte> buffer) => m_File.Read(buffer);
-
-        public void Close()
         {
-            m_File.Close();
+            var length = buffer.Length;
-        }
+            if (length > _segment.Count - Position)
+            {
+                throw new OutOfMemoryException();
+            }
 
-        public long Seek(long offset, SeekOrigin origin) => m_File.BaseStream.Seek(offset, origin);
+            _segment.AsSpan(Position, length).CopyTo(buffer);
+            Position += length;
+            return length;
+        }
     }
 }

Projects/Server/Serialization/BufferWriter.cs

@@ -32,8 +32,6 @@ namespace Server
 
         protected long Index { get; set; }
 
-        private readonly char[] m_SingleCharBuffer = new char[1];
-
         private byte[] m_CharacterBuffer;
 
         private int m_MaxBufferChars;
@@ -190,7 +188,7 @@ namespace Server
         {
             var bits = decimal.GetBits(value);
 
-            for (var i = 0; i < bits.Length; ++i)
+            for (var i = 0; i < 4; ++i)
             {
                 Write(bits[i]);
             }
@@ -314,19 +312,6 @@ namespace Server
             Index += 4;
         }
 
-        public void Write(char value)
-        {
-            if (Index + 8 > Buffer.Length)
-            {
-                Flush();
-            }
-
-            m_SingleCharBuffer[0] = value;
-
-            var byteCount = m_Encoding.GetBytes(m_SingleCharBuffer, 0, 1, Buffer, (int)Index);
-            Index += byteCount;
-        }
-
         public void Write(byte value)
         {
             if (Index + 1 > Buffer.Length)
@@ -769,7 +754,7 @@ namespace Server
 
                 while (charsLeft > 0)
                 {
-                    var charCount = charsLeft > m_MaxBufferChars ? m_MaxBufferChars : charsLeft;
+                    var charCount = Math.Min(charsLeft, m_MaxBufferChars);
                     var byteLength = m_Encoding.GetBytes(value, current, charCount, m_CharacterBuffer, 0);
 
                     if (Index + byteLength > Buffer.Length)

Projects/Server/Serialization/IGenericReader.cs

@@ -36,7 +36,6 @@ namespace Server
         ushort ReadUShort();
         double ReadDouble();
         float ReadFloat();
-        char ReadChar();
         byte ReadByte();
         sbyte ReadSByte();
         bool ReadBool();
@@ -67,7 +66,6 @@ namespace Server
         HashSet<BaseGuild> ReadGuildSet();
         HashSet<T> ReadGuildSet<T>() where T : BaseGuild;
         Race ReadRace();
-        bool End();
         int Read(Span<byte> buffer);
     }
 }

Projects/Server/Serialization/IGenericWriter.cs

@@ -38,7 +38,6 @@ namespace Server
         void Write(ushort value);
         void Write(double value);
         void Write(float value);
-        void Write(char value);
         void Write(byte value);
         void Write(byte[] value, int length);
         void Write(sbyte value);

Projects/Server/World/World.cs

@@ -14,6 +14,7 @@
  *************************************************************************/
 
 using System;
+using System.Buffers;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -31,6 +32,7 @@ namespace Server
     {
         Initial,
         Loading,
+        WritingLoadBuffers,
         Running,
         Saving,
         WritingSave
@@ -294,6 +296,35 @@ namespace Server
             return map;
         }
 
+        private static void SaveBuffers<I, T>(IIndexInfo<I> indexInfo, List<EntityIndex<T>> entities) where T : ISerializable
+        {
+            var indexType = indexInfo.TypeName;
+
+            string dataPath = Path.Combine("Saves", indexType, $"{indexType}.bin");
+
+            if (!File.Exists(dataPath))
+            {
+                return;
+            }
+
+            using FileStream bin = new FileStream(dataPath, FileMode.Open, FileAccess.Read, FileShare.Read);
+            BinaryReader reader = new BinaryReader(bin);
+
+            foreach (var entry in entities)
+            {
+                T t = entry.Entity;
+
+                if (t == null || t is IEntity entity && entity.Deleted)
+                {
+                    continue;
+                }
+
+                byte[] saveBuffer = new byte[entry.Length];
+                reader.Read(saveBuffer, 0, entry.Length);
+                t.SaveBuffer = new BufferWriter(saveBuffer, true);
+            }
+        }
+
         private static void LoadData<I, T>(IIndexInfo<I> indexInfo, List<EntityIndex<T>> entities) where T : ISerializable
         {
             var indexType = indexInfo.TypeName;
@@ -306,7 +337,11 @@ namespace Server
             }
 
             using FileStream bin = new FileStream(dataPath, FileMode.Open, FileAccess.Read, FileShare.Read);
-            BinaryFileReader reader = new BinaryFileReader(new BinaryReader(bin));
+            var fileBuffer = new byte[bin.Length];
+            bin.Read(fileBuffer);
+            bin.Close();
+
+            int position = 0;
 
             foreach (var entry in entities)
             {
@@ -317,15 +352,15 @@ namespace Server
                     continue;
                 }
 
-                reader.Seek(entry.Position, SeekOrigin.Begin);
+                var segment = new ArraySegment<byte>(fileBuffer, position, entry.Length);
+                var bufferReader = new BufferReader(segment);
+                t.Deserialize(bufferReader);
 
-                t.Deserialize(reader);
+                if (bufferReader.Position != entry.Length)
-
-                if (reader.Position != entry.Position + entry.Length)
                 {
                     Console.WriteLine($"***** Bad deserialize on {t.GetType()} *****");
                     Console.WriteLine(
-                        $"Serialized object was {entry.Length} bytes, but {reader.Position - entry.Position} bytes deserialized"
+                        $"Serialized object was {entry.Length} bytes, but {bufferReader.Position - entry.Position} bytes deserialized"
                     );
 
                     Console.WriteLine("Delete the object and continue? (y/n)");
@@ -337,13 +372,9 @@ namespace Server
                     t.Delete();
                 }
 
-                reader.Seek(entry.Position, SeekOrigin.Begin);
 
-                t.SaveBuffer = new BufferWriter(new byte[entry.Length], true);
+                position += entry.Length;
-                reader.Read(t.SaveBuffer.Buffer);
             }
-
-            reader.Close();
         }
 
         public static void Load()
@@ -377,7 +408,7 @@ namespace Server
 
             EventSink.InvokeWorldLoad();
 
-            WorldState = WorldState.Running;
+            WorldState = WorldState.WritingLoadBuffers;
 
             ProcessSafetyQueues();
 
@@ -407,6 +438,16 @@ namespace Server
                 Items.Count,
                 Mobiles.Count
             );
+
+            // Async save buffers
+            ThreadPool.QueueUserWorkItem(state =>
+                {
+                    SaveBuffers(mobileIndexInfo, mobiles);
+                    SaveBuffers(itemIndexInfo, items);
+                    SaveBuffers(guildIndexInfo, guilds);
+                    WorldState = WorldState.Running;
+                }
+            );
         }
 
         private static void ProcessSafetyQueues()
@@ -457,7 +498,6 @@ namespace Server
 
         public static void WriteFiles(object state)
         {
-            Console.Write("Closing Save Files...");
             var watch = Stopwatch.StartNew();
 
             IIndexInfo<Serial> itemIndexInfo = new EntityTypeIndex("Items");
@@ -472,7 +512,7 @@ namespace Server
 
             m_DiskWriteHandle.Set();
 
-            Console.WriteLine("done {0:F1} seconds.", watch.Elapsed.TotalSeconds);
+            Console.WriteLine("World: Writing snapshot took {0:F1} seconds.", watch.Elapsed.TotalSeconds);
 
             Timer.DelayCall(FinishWorldSave);
         }
@@ -626,6 +666,7 @@ namespace Server
 
                         goto case WorldState.Running;
                     }
+                case WorldState.WritingLoadBuffers:
                 case WorldState.Running:
                     {
                         if (serial.IsItem)
@@ -654,7 +695,6 @@ namespace Server
 
         public static BaseGuild FindGuild(Serial serial) => Guilds.TryGetValue(serial, out var guild) ? guild : null;
 
-
         public static void AddEntity<T>(T entity) where T : class, IEntity
         {
             switch (WorldState)
@@ -680,6 +720,7 @@ namespace Server
                         _pendingAdd[entity.Serial] = entity;
                         break;
                     }
+                case WorldState.WritingLoadBuffers:
                 case WorldState.Running:
                     {
                         if (entity.Serial.IsItem)
@@ -718,6 +759,7 @@ namespace Server
                         _pendingDelete[entity.Serial] = entity;
                         break;
                     }
+                case WorldState.WritingLoadBuffers:
                 case WorldState.Running:
                     {
                         if (entity.Serial.IsItem)

version.json

@@ -1,4 +1,8 @@
 {
   "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json",
-  "version": "0.8.1"
+  "version": "0.8.1",
+  "cloudBuild": {
+    "setVersionVariables": false,
+    "setAllVariables": false
+  }
 }