From 313eda6a3e4223eab3e0c2e9f270a6f99687505b Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Mon, 16 Nov 2020 21:58:58 -0800 Subject: [PATCH] Speeds up world loading (#315) - [X] Speeds up world loading - [X] Removes PeekChar cause it wasn't used - [X] Adds a BufferReader - [X] Removes BinaryFileReader - [X] Removes reading/writing `char` since it is not consistent and will cause a deserialize issue if used. Bumps version release --- Projects/Server/Geometry/Rectangle2D.cs | 6 + .../{BinaryFileReader.cs => BufferReader.cs} | 160 ++++++++++++------ Projects/Server/Serialization/BufferWriter.cs | 19 +-- .../Server/Serialization/IGenericReader.cs | 2 - .../Server/Serialization/IGenericWriter.cs | 1 - Projects/Server/World/World.cs | 72 ++++++-- version.json | 6 +- 7 files changed, 175 insertions(+), 91 deletions(-) rename Projects/Server/Serialization/{BinaryFileReader.cs => BufferReader.cs} (64%) diff --git a/Projects/Server/Geometry/Rectangle2D.cs b/Projects/Server/Geometry/Rectangle2D.cs index 48a42aaec..2ad4e1225 100644 --- a/Projects/Server/Geometry/Rectangle2D.cs +++ b/Projects/Server/Geometry/Rectangle2D.cs @@ -29,6 +29,12 @@ namespace Server m_End = new Point2D(end); } + public Rectangle2D(Point2D start, Point2D end) + { + m_Start = start; + m_End = end; + } + public Rectangle2D(int x, int y, int width, int height) { m_Start = new Point2D(x, y); diff --git a/Projects/Server/Serialization/BinaryFileReader.cs b/Projects/Server/Serialization/BufferReader.cs similarity index 64% rename from Projects/Server/Serialization/BinaryFileReader.cs rename to Projects/Server/Serialization/BufferReader.cs index e720da180..593d51ab5 100644 --- a/Projects/Server/Serialization/BinaryFileReader.cs +++ b/Projects/Server/Serialization/BufferReader.cs @@ -2,7 +2,7 @@ * ModernUO * * Copyright 2019-2020 - ModernUO Development Team * * Email: hi@modernuo.com * - * File: BinaryFileReader.cs * + * File: BufferReader.cs * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * @@ -14,26 +14,48 @@ *************************************************************************/ using System; +using System.Buffers.Binary; using System.Collections.Generic; -using System.IO; using System.Net; +using System.Text; using Server.Guilds; namespace Server { - public sealed class BinaryFileReader : IGenericReader + public class BufferReader : IGenericReader { - private readonly BinaryReader m_File; + private readonly Encoding _encoding; + private ArraySegment _segment; + public int Position { get; private set; } - public BinaryFileReader(BinaryReader br) => m_File = br; + public BufferReader(ArraySegment segment) + { + _segment = segment; + _encoding = Utility.UTF8; + } - public long Position => m_File.BaseStream.Position; + public string ReadString() + { + if (!ReadBool()) + { + return null; + } - public string ReadString() => ReadByte() != 0 ? m_File.ReadString() : null; + var length = ReadEncodedInt(); + var s = length == 0 ? "" : _encoding.GetString(_segment.AsSpan(Position, length)); + Position += length; + return s; + } + + public DateTime ReadDateTime() => new DateTime(ReadLong()); + + public DateTimeOffset ReadDateTimeOffset() => new DateTimeOffset(ReadLong(), ReadTimeSpan()); + + public TimeSpan ReadTimeSpan() => new TimeSpan(ReadLong()); public DateTime ReadDeltaTime() { - var ticks = m_File.ReadInt64(); + var ticks = ReadLong(); var now = DateTime.UtcNow.Ticks; if (ticks > 0 && ticks + now < 0) @@ -56,7 +78,69 @@ namespace Server } } - public IPAddress ReadIPAddress() => new IPAddress(m_File.ReadInt64()); + public decimal ReadDecimal() => new decimal(new[] { ReadInt(), ReadInt(), ReadInt(), ReadInt() }); + + public long ReadLong() + { + var v = BinaryPrimitives.ReadInt64LittleEndian(_segment.AsSpan(Position, 8)); + Position += 8; + return v; + } + + public ulong ReadULong() + { + var v = BinaryPrimitives.ReadUInt64LittleEndian(_segment.AsSpan(Position, 8)); + Position += 8; + return v; + } + + public int ReadInt() + { + var v = BinaryPrimitives.ReadInt32LittleEndian(_segment.AsSpan(Position, 4)); + Position += 4; + return v; + } + + public uint ReadUInt() + { + var v = BinaryPrimitives.ReadUInt32LittleEndian(_segment.AsSpan(Position, 4)); + Position += 4; + return v; + } + + public short ReadShort() + { + var v = BinaryPrimitives.ReadInt16LittleEndian(_segment.AsSpan(Position, 2)); + Position += 2; + return v; + } + + public ushort ReadUShort() + { + var v = BinaryPrimitives.ReadUInt16LittleEndian(_segment.AsSpan(Position, 2)); + Position += 2; + return v; + } + + public double ReadDouble() + { + var v = BinaryPrimitives.ReadDoubleLittleEndian(_segment.AsSpan(Position, 8)); + Position += 8; + return v; + } + + public float ReadFloat() + { + var v = BinaryPrimitives.ReadSingleLittleEndian(_segment.AsSpan(Position, 4)); + Position += 4; + return v; + } + + public byte ReadByte() => _segment[Position++]; + + public sbyte ReadSByte() => (sbyte)_segment[Position++]; + + public bool ReadBool() => _segment[Position++] != 0; public int ReadEncodedInt() { @@ -65,7 +149,7 @@ namespace Server do { - b = m_File.ReadByte(); + b = ReadByte(); v |= (b & 0x7F) << shift; shift += 7; } while (b >= 0x80); @@ -73,43 +157,7 @@ namespace Server return v; } - public DateTime ReadDateTime() => new DateTime(m_File.ReadInt64()); - - public DateTimeOffset ReadDateTimeOffset() - { - var ticks = m_File.ReadInt64(); - var offset = new TimeSpan(m_File.ReadInt64()); - - return new DateTimeOffset(ticks, offset); - } - - public TimeSpan ReadTimeSpan() => new TimeSpan(m_File.ReadInt64()); - - public decimal ReadDecimal() => m_File.ReadDecimal(); - - public long ReadLong() => m_File.ReadInt64(); - - public ulong ReadULong() => m_File.ReadUInt64(); - - public int ReadInt() => m_File.ReadInt32(); - - public uint ReadUInt() => m_File.ReadUInt32(); - - public short ReadShort() => m_File.ReadInt16(); - - public ushort ReadUShort() => m_File.ReadUInt16(); - - public double ReadDouble() => m_File.ReadDouble(); - - public float ReadFloat() => m_File.ReadSingle(); - - public char ReadChar() => m_File.ReadChar(); - - public byte ReadByte() => m_File.ReadByte(); - - public sbyte ReadSByte() => m_File.ReadSByte(); - - public bool ReadBool() => m_File.ReadBoolean(); + public IPAddress ReadIPAddress() => new IPAddress(ReadLong()); public Point3D ReadPoint3D() => new Point3D(ReadInt(), ReadInt(), ReadInt()); @@ -259,15 +307,17 @@ namespace Server public Race ReadRace() => Race.Races[ReadByte()]; - public bool End() => m_File.PeekChar() == -1; - - public int Read(Span buffer) => m_File.Read(buffer); - - public void Close() + public int Read(Span buffer) { - m_File.Close(); - } + var length = buffer.Length; + if (length > _segment.Count - Position) + { + throw new OutOfMemoryException(); + } - public long Seek(long offset, SeekOrigin origin) => m_File.BaseStream.Seek(offset, origin); + _segment.AsSpan(Position, length).CopyTo(buffer); + Position += length; + return length; + } } } diff --git a/Projects/Server/Serialization/BufferWriter.cs b/Projects/Server/Serialization/BufferWriter.cs index 84081b150..74ff4668e 100644 --- a/Projects/Server/Serialization/BufferWriter.cs +++ b/Projects/Server/Serialization/BufferWriter.cs @@ -32,8 +32,6 @@ namespace Server protected long Index { get; set; } - private readonly char[] m_SingleCharBuffer = new char[1]; - private byte[] m_CharacterBuffer; private int m_MaxBufferChars; @@ -190,7 +188,7 @@ namespace Server { var bits = decimal.GetBits(value); - for (var i = 0; i < bits.Length; ++i) + for (var i = 0; i < 4; ++i) { Write(bits[i]); } @@ -314,19 +312,6 @@ namespace Server Index += 4; } - public void Write(char value) - { - if (Index + 8 > Buffer.Length) - { - Flush(); - } - - m_SingleCharBuffer[0] = value; - - var byteCount = m_Encoding.GetBytes(m_SingleCharBuffer, 0, 1, Buffer, (int)Index); - Index += byteCount; - } - public void Write(byte value) { if (Index + 1 > Buffer.Length) @@ -769,7 +754,7 @@ namespace Server while (charsLeft > 0) { - var charCount = charsLeft > m_MaxBufferChars ? m_MaxBufferChars : charsLeft; + var charCount = Math.Min(charsLeft, m_MaxBufferChars); var byteLength = m_Encoding.GetBytes(value, current, charCount, m_CharacterBuffer, 0); if (Index + byteLength > Buffer.Length) diff --git a/Projects/Server/Serialization/IGenericReader.cs b/Projects/Server/Serialization/IGenericReader.cs index 1fcbfab56..609e28626 100644 --- a/Projects/Server/Serialization/IGenericReader.cs +++ b/Projects/Server/Serialization/IGenericReader.cs @@ -36,7 +36,6 @@ namespace Server ushort ReadUShort(); double ReadDouble(); float ReadFloat(); - char ReadChar(); byte ReadByte(); sbyte ReadSByte(); bool ReadBool(); @@ -67,7 +66,6 @@ namespace Server HashSet ReadGuildSet(); HashSet ReadGuildSet() where T : BaseGuild; Race ReadRace(); - bool End(); int Read(Span buffer); } } diff --git a/Projects/Server/Serialization/IGenericWriter.cs b/Projects/Server/Serialization/IGenericWriter.cs index 052027690..3fcbdbbce 100644 --- a/Projects/Server/Serialization/IGenericWriter.cs +++ b/Projects/Server/Serialization/IGenericWriter.cs @@ -38,7 +38,6 @@ namespace Server void Write(ushort value); void Write(double value); void Write(float value); - void Write(char value); void Write(byte value); void Write(byte[] value, int length); void Write(sbyte value); diff --git a/Projects/Server/World/World.cs b/Projects/Server/World/World.cs index b2aa02afb..87e67db37 100644 --- a/Projects/Server/World/World.cs +++ b/Projects/Server/World/World.cs @@ -14,6 +14,7 @@ *************************************************************************/ using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -31,6 +32,7 @@ namespace Server { Initial, Loading, + WritingLoadBuffers, Running, Saving, WritingSave @@ -294,6 +296,35 @@ namespace Server return map; } + private static void SaveBuffers(IIndexInfo indexInfo, List> entities) where T : ISerializable + { + var indexType = indexInfo.TypeName; + + string dataPath = Path.Combine("Saves", indexType, $"{indexType}.bin"); + + if (!File.Exists(dataPath)) + { + return; + } + + using FileStream bin = new FileStream(dataPath, FileMode.Open, FileAccess.Read, FileShare.Read); + BinaryReader reader = new BinaryReader(bin); + + foreach (var entry in entities) + { + T t = entry.Entity; + + if (t == null || t is IEntity entity && entity.Deleted) + { + continue; + } + + byte[] saveBuffer = new byte[entry.Length]; + reader.Read(saveBuffer, 0, entry.Length); + t.SaveBuffer = new BufferWriter(saveBuffer, true); + } + } + private static void LoadData(IIndexInfo indexInfo, List> entities) where T : ISerializable { var indexType = indexInfo.TypeName; @@ -306,7 +337,11 @@ namespace Server } using FileStream bin = new FileStream(dataPath, FileMode.Open, FileAccess.Read, FileShare.Read); - BinaryFileReader reader = new BinaryFileReader(new BinaryReader(bin)); + var fileBuffer = new byte[bin.Length]; + bin.Read(fileBuffer); + bin.Close(); + + int position = 0; foreach (var entry in entities) { @@ -317,15 +352,15 @@ namespace Server continue; } - reader.Seek(entry.Position, SeekOrigin.Begin); + var segment = new ArraySegment(fileBuffer, position, entry.Length); + var bufferReader = new BufferReader(segment); + t.Deserialize(bufferReader); - t.Deserialize(reader); - - if (reader.Position != entry.Position + entry.Length) + if (bufferReader.Position != entry.Length) { Console.WriteLine($"***** Bad deserialize on {t.GetType()} *****"); Console.WriteLine( - $"Serialized object was {entry.Length} bytes, but {reader.Position - entry.Position} bytes deserialized" + $"Serialized object was {entry.Length} bytes, but {bufferReader.Position - entry.Position} bytes deserialized" ); Console.WriteLine("Delete the object and continue? (y/n)"); @@ -337,13 +372,9 @@ namespace Server t.Delete(); } - reader.Seek(entry.Position, SeekOrigin.Begin); - t.SaveBuffer = new BufferWriter(new byte[entry.Length], true); - reader.Read(t.SaveBuffer.Buffer); + position += entry.Length; } - - reader.Close(); } public static void Load() @@ -377,7 +408,7 @@ namespace Server EventSink.InvokeWorldLoad(); - WorldState = WorldState.Running; + WorldState = WorldState.WritingLoadBuffers; ProcessSafetyQueues(); @@ -407,6 +438,16 @@ namespace Server Items.Count, Mobiles.Count ); + + // Async save buffers + ThreadPool.QueueUserWorkItem(state => + { + SaveBuffers(mobileIndexInfo, mobiles); + SaveBuffers(itemIndexInfo, items); + SaveBuffers(guildIndexInfo, guilds); + WorldState = WorldState.Running; + } + ); } private static void ProcessSafetyQueues() @@ -457,7 +498,6 @@ namespace Server public static void WriteFiles(object state) { - Console.Write("Closing Save Files..."); var watch = Stopwatch.StartNew(); IIndexInfo itemIndexInfo = new EntityTypeIndex("Items"); @@ -472,7 +512,7 @@ namespace Server m_DiskWriteHandle.Set(); - Console.WriteLine("done {0:F1} seconds.", watch.Elapsed.TotalSeconds); + Console.WriteLine("World: Writing snapshot took {0:F1} seconds.", watch.Elapsed.TotalSeconds); Timer.DelayCall(FinishWorldSave); } @@ -626,6 +666,7 @@ namespace Server goto case WorldState.Running; } + case WorldState.WritingLoadBuffers: case WorldState.Running: { if (serial.IsItem) @@ -654,7 +695,6 @@ namespace Server public static BaseGuild FindGuild(Serial serial) => Guilds.TryGetValue(serial, out var guild) ? guild : null; - public static void AddEntity(T entity) where T : class, IEntity { switch (WorldState) @@ -680,6 +720,7 @@ namespace Server _pendingAdd[entity.Serial] = entity; break; } + case WorldState.WritingLoadBuffers: case WorldState.Running: { if (entity.Serial.IsItem) @@ -718,6 +759,7 @@ namespace Server _pendingDelete[entity.Serial] = entity; break; } + case WorldState.WritingLoadBuffers: case WorldState.Running: { if (entity.Serial.IsItem) diff --git a/version.json b/version.json index 403de5449..2efc4f425 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,8 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "0.8.1" + "version": "0.8.1", + "cloudBuild": { + "setVersionVariables": false, + "setAllVariables": false + } }