From e1e30998ba0c8dd005e75cc6a420488746a77976 Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Tue, 11 Oct 2022 22:17:22 -0700 Subject: [PATCH] fix: Adds ReadType/Write(Type) and improves type referencing (#1172) ## Changes * Improves type hashing by introducing xxHash3 (64bit) * Removes individual `tdb` files in favor of a single `SerializedTypes.db` file. This file is only used to identify a type that is being deserialized, which doesn't exist. * Adds duplicate type alias detection * Adds `AssemblyHandler.FindTypeByHash` View changed files whitespaces: https://github.com/modernuo/ModernUO/pull/1172/files?diff=split&w=1 ## SerializedTypes.db The serialized types file is used to get back the original name of a type in case it no longer exists in code. This can easily be necessary if a class is renamed in code and no `TypeAlias` is provided. ### Format byte[4] - version byte[4] - count --array-- byte[8] - xxHash byte[1] - flag, 0 - null, 1 - not null byte[n] - Full class name in UTF8 ### Example SerializedTypes_Example ## Benchmarks Serialized 500 Type fields. The 8192bytes comes from the _ConcurrentQueue_ that would later be used for SerializedTypes. Note that the queue is never cleared, so it's size grew considerably. ```cs | Method | Mean | Error | StdDev | Allocated | |--------------------- |---------:|---------:|---------:|----------:| | BenchmarkXXHash | 18.44 us | 0.278 us | 0.260 us | 8192 B | | BenchmarkTypeStrings | 25.09 us | 0.292 us | 0.259 us | - | ``` TODO: * Add support in the Serialization Generator for `ReadType()` and `Write(Type)` * Remove `SetTypeRef` from Serialization Generator --- Projects/Server/AssemblyHandler.cs | 153 ++++++++++------ Projects/Server/Guild.cs | 10 -- Projects/Server/IEntity.cs | 4 - Projects/Server/Items/Container.cs | 5 +- Projects/Server/Items/Item.cs | 11 -- Projects/Server/Mobiles/Mobile.cs | 12 -- .../Server/Serialization/AdhocPersistence.cs | 45 +++-- .../Server/Serialization/BinaryFileReader.cs | 80 +++++++-- .../Server/Serialization/BinaryFileWriter.cs | 7 +- Projects/Server/Serialization/BufferReader.cs | 84 +++++++-- Projects/Server/Serialization/BufferWriter.cs | 59 +++++-- .../Serialization/GenericPersistence.cs | 12 +- .../Server/Serialization/IGenericReader.cs | 1 + .../Server/Serialization/IGenericWriter.cs | 1 + .../Server/Serialization/ISerializable.cs | 12 +- Projects/Server/Serialization/Persistence.cs | 66 ++++++- Projects/Server/Server.csproj | 1 + Projects/Server/Utilities/HashUtility.cs | 48 +++++ Projects/Server/Utilities/Utility.cs | 4 +- Projects/Server/World/EntityPersistence.cs | 166 ++++++++++-------- Projects/Server/World/EntitySpan.cs | 5 +- Projects/Server/World/World.cs | 79 ++++++--- Projects/UOContent/Accounting/Account.cs | 18 +- Projects/UOContent/Accounting/Accounts.cs | 18 +- .../Armor/Cloth/GargishClothKiltType2.cs | 1 - .../Monsters/Reptile/Magic/SkeletalDragon.cs | 2 - .../Spells/Chivalry/NobleSacrifice.cs | 1 - version.json | 2 +- 28 files changed, 616 insertions(+), 291 deletions(-) create mode 100644 Projects/Server/Utilities/HashUtility.cs diff --git a/Projects/Server/AssemblyHandler.cs b/Projects/Server/AssemblyHandler.cs index fcb2866a8..9ba3da8eb 100644 --- a/Projects/Server/AssemblyHandler.cs +++ b/Projects/Server/AssemblyHandler.cs @@ -14,12 +14,14 @@ *************************************************************************/ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Runtime.Loader; +using Server.Logging; namespace Server; @@ -27,6 +29,7 @@ public static class AssemblyHandler { private static readonly Dictionary m_TypeCaches = new(); private static TypeCache m_NullCache; + public static Assembly[] Assemblies { get; set; } internal static Assembly AssemblyResolver(object sender, ResolveEventArgs args) @@ -66,6 +69,8 @@ public static class AssemblyHandler EnsureAssemblyDirectories(); var assemblyDirectories = ServerConfiguration.AssemblyDirectories; + Assembly assembly = null; + foreach (var assemblyDir in assemblyDirectories) { var assemblyPath = PathUtility.GetFullPath(Path.Combine(assemblyDir, fileName), Core.BaseDirectory); @@ -74,12 +79,17 @@ public static class AssemblyHandler var assemblyNameCheck = AssemblyName.GetAssemblyName(assemblyPath); if (assemblyNameCheck.FullName == fullName) { - return AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath); + assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath); + break; } } } - return null; + // This forces the type caching to be generated. + // We need this for world loading to find types by hash. + GetTypeCache(assembly); + + return assembly; } public static Assembly LoadAssemblyByFileName(string assemblyFile) @@ -154,6 +164,11 @@ public static class AssemblyHandler } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ulong GetTypeHash(Type type) => GetTypeHash(type.FullName); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ulong GetTypeHash(string key) => key == null ? 0 : HashUtility.ComputeHash64(key); public static TypeCache GetTypeCache(Assembly asm) { @@ -195,84 +210,120 @@ public static class AssemblyHandler return null; } + + public static Type FindTypeByHash(ulong hash) + { + for (var i = 0; i < Assemblies.Length; i++) + { + foreach (var type in GetTypeCache(Assemblies[i]).GetTypesByHash(hash, true, false)) + { + return type; + } + } + + foreach(var type in GetTypeCache(Core.Assembly).GetTypesByHash(hash, true, false)) + { + return type; + } + + return null; + } } public class TypeCache { - private readonly Dictionary _nameMap = new(); - private readonly Dictionary _nameMapInsensitive = new(); - private readonly Dictionary _fullNameMap = new(); - private readonly Dictionary _fullNameMapInsensitive = new(); + private static ILogger logger = LogFactory.GetLogger(typeof(TypeCache)); + + private Dictionary _nameMap = new(); + private Dictionary _nameMapInsensitive = new(); + private Dictionary _fullNameMap = new(); + private Dictionary _fullNameMapInsensitive = new(); public TypeCache(Assembly asm) { Types = asm?.GetTypes() ?? Type.EmptyTypes; - var nameMap = new Dictionary>(); - var nameMapInsensitive = new Dictionary>(); - var fullNameMap = new Dictionary>(); - var fullNameMapInsensitive = new Dictionary>(); + var nameMap = new Dictionary>(); + var nameMapInsensitive = new Dictionary>(); + var fullNameMap = new Dictionary>(); + var fullNameMapInsensitive = new Dictionary>(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - void addTypeToRefs(int index, string fullTypeName) + void addTypeToRefs(Type type, string typeName, string fullTypeName) { - var typeName = fullTypeName[(fullTypeName.LastIndexOf('.') + 1)..]; - AddToRefs(index, typeName, nameMap); - AddToRefs(index, typeName.ToLower(), nameMapInsensitive); - AddToRefs(index, fullTypeName, fullNameMap); - AddToRefs(index, fullTypeName.ToLower(), fullNameMapInsensitive); + AddToRefs(type, typeName, nameMap); + AddToRefs(type, typeName.ToLower(), nameMapInsensitive); + AddToRefs(type, fullTypeName, fullNameMap); + AddToRefs(type, fullTypeName.ToLower(), fullNameMapInsensitive); } var aliasType = typeof(TypeAliasAttribute); for (var i = 0; i < Types.Length; i++) { var current = Types[i]; - addTypeToRefs(i, current.FullName); + addTypeToRefs(current, current.Name, current.FullName ?? ""); if (current.GetCustomAttribute(aliasType, false) is TypeAliasAttribute alias) { for (var j = 0; j < alias.Aliases.Length; j++) { - addTypeToRefs(i, alias.Aliases[j]); + var fullTypeName = alias.Aliases[j]; + var typeName = fullTypeName[(fullTypeName.LastIndexOf('.')+1)..]; + addTypeToRefs(current, typeName, fullTypeName); } } } foreach (var (key, value) in nameMap) { - _nameMap[key] = value.ToArray(); + _nameMap[HashUtility.ComputeHash64(key)] = value.ToArray(); } foreach (var (key, value) in nameMapInsensitive) { - _nameMapInsensitive[key] = value.ToArray(); + _nameMapInsensitive[HashUtility.ComputeHash64(key)] = value.ToArray(); } foreach (var (key, value) in fullNameMap) { - _fullNameMap[key] = value.ToArray(); + var values = value.ToArray(); + _fullNameMap[HashUtility.ComputeHash64(key)] = value.ToArray(); +#if DEBUG + if (values.Length > 1) + { + for (var i = 0; i < values.Length; i++) + { + var type = values[i]; + logger.Warning( + "Duplicate type {Type1} for {Name}.", + type, + key + ); + } + } +#endif } foreach (var (key, value) in fullNameMapInsensitive) { - _fullNameMapInsensitive[key] = value.ToArray(); + _fullNameMapInsensitive[HashUtility.ComputeHash64(key)] = value.ToArray(); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void AddToRefs(int index, string key, Dictionary> map) + private static void AddToRefs(Type type, string key, Dictionary> map) { - if (key == null) + if (string.IsNullOrEmpty(key)) { return; } if (map.TryGetValue(key, out var refs)) { - refs.Add(index); + refs.Add(type); } else { - refs = new HashSet { index }; + refs = new HashSet { type }; map.Add(key, refs); } } @@ -280,64 +331,50 @@ public class TypeCache public Type[] Types { get; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public TypeEnumerable GetTypesByName(string name, bool full, bool ignoreCase) => new(name, this, full, ignoreCase); + public TypeEnumerator GetTypesByName(string name, bool full, bool ignoreCase) => new(name, this, full, ignoreCase); - public ref struct TypeEnumerable - { - private readonly TypeCache _cache; - private readonly string _name; - private readonly bool _ignoreCase; - private readonly bool _full; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public TypeEnumerable(string name, TypeCache cache, bool full, bool ignoreCase) - { - _name = name; - _cache = cache; - _ignoreCase = ignoreCase; - _full = full; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public TypeEnumerator GetEnumerator() => new(_name, _cache, _full, _ignoreCase); - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public TypeEnumerator GetTypesByHash(ulong hash, bool full, bool ignoreCase) => new(hash, this, full, ignoreCase); public ref struct TypeEnumerator { - private readonly TypeCache _cache; - private readonly int[] _values; + private Type[] _values; private int _index; private Type _current; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal TypeEnumerator(string name, TypeCache cache, bool full, bool ignoreCase) + : this(HashUtility.ComputeHash64(ignoreCase ? name.ToLower() : name), cache, full, ignoreCase) { - _cache = cache; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal TypeEnumerator(ulong hash, TypeCache cache, bool full, bool ignoreCase) + { if (ignoreCase) { - var map = full ? _cache._fullNameMapInsensitive : _cache._nameMapInsensitive; - _values = map.TryGetValue(name.ToLower(), out var values) ? values : Array.Empty(); + var map = full ? cache._fullNameMapInsensitive : cache._nameMapInsensitive; + _values = map.TryGetValue(hash, out var values) ? values : Array.Empty(); } else { - var map = full ? _cache._fullNameMap : _cache._nameMap; - _values = map.TryGetValue(name, out var values) ? values : Array.Empty(); + var map = full ? cache._fullNameMap : cache._nameMap; + _values = map.TryGetValue(hash, out var values) ? values : Array.Empty(); } _index = 0; _current = default; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public TypeEnumerator GetEnumerator() => this; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool MoveNext() { - int[] localList = _values; - - if ((uint)_index < (uint)localList.Length) + if ((uint)_index < (uint)_values.Length) { - _current = _cache.Types[_values[_index++]]; - + _current = _values[_index++]; return true; } diff --git a/Projects/Server/Guild.cs b/Projects/Server/Guild.cs index b006864d4..2e164b918 100644 --- a/Projects/Server/Guild.cs +++ b/Projects/Server/Guild.cs @@ -31,25 +31,15 @@ public abstract class BaseGuild : ISerializable { Serial = World.NewGuild; World.AddGuild(this); - - SetTypeRef(GetType()); } protected BaseGuild(Serial serial) { Serial = serial; - SetTypeRef(GetType()); } public void SetTypeRef(Type type) { - TypeRef = World.GuildTypes.IndexOf(type); - - if (TypeRef == -1) - { - World.GuildTypes.Add(type); - TypeRef = World.GuildTypes.Count - 1; - } } public abstract string Abbreviation { get; set; } diff --git a/Projects/Server/IEntity.cs b/Projects/Server/IEntity.cs index 0d1614332..b0335c555 100644 --- a/Projects/Server/IEntity.cs +++ b/Projects/Server/IEntity.cs @@ -43,10 +43,6 @@ public class Entity : IEntity public Entity(Serial serial) => Serial = serial; - public void SetTypeRef(Type type) - { - } - DateTime ISerializable.Created { get; set; } = Core.Now; DateTime ISerializable.LastSerialized { get; set; } = DateTime.MaxValue; diff --git a/Projects/Server/Items/Container.cs b/Projects/Server/Items/Container.cs index e1eeb8cab..f9d3fca46 100644 --- a/Projects/Server/Items/Container.cs +++ b/Projects/Server/Items/Container.cs @@ -1809,8 +1809,7 @@ public class Container : Item public class ContainerData { - private static ILogger _logger; - private static ILogger Logger => _logger ??= LogFactory.GetLogger(typeof(ContainerData)); + private static ILogger logger = LogFactory.GetLogger(typeof(ContainerData)); private static readonly Dictionary m_Table; static ContainerData() @@ -1875,7 +1874,7 @@ public class ContainerData if (m_Table.ContainsKey(id)) { - Logger.Warning("double ItemID entry in Data\\containers.cfg"); + logger.Warning("double ItemID entry in Data\\containers.cfg"); } else { diff --git a/Projects/Server/Items/Item.cs b/Projects/Server/Items/Item.cs index a85317de4..66f78ecf2 100644 --- a/Projects/Server/Items/Item.cs +++ b/Projects/Server/Items/Item.cs @@ -217,24 +217,15 @@ public class Item : IHued, IComparable, ISpawnable, IObjectPropertyListEnt SetLastMoved(); World.AddEntity(this); - SetTypeRef(GetType()); } public Item(Serial serial) { Serial = serial; - SetTypeRef(GetType()); } public void SetTypeRef(Type type) { - TypeRef = World.ItemTypes.IndexOf(type); - - if (TypeRef == -1) - { - World.ItemTypes.Add(type); - TypeRef = World.ItemTypes.Count - 1; - } } public int TempFlags @@ -786,8 +777,6 @@ public class Item : IHued, IComparable, ISpawnable, IObjectPropertyListEnt [CommandProperty(AccessLevel.Counselor)] public Serial Serial { get; } - public int TypeRef { get; private set; } - public virtual void Serialize(IGenericWriter writer) { writer.Write(9); // version diff --git a/Projects/Server/Mobiles/Mobile.cs b/Projects/Server/Mobiles/Mobile.cs index 6425b9e4f..fd1799d98 100644 --- a/Projects/Server/Mobiles/Mobile.cs +++ b/Projects/Server/Mobiles/Mobile.cs @@ -340,7 +340,6 @@ public class Mobile : IHued, IComparable, ISpawnable, IObjectPropertyLis DefaultMobileInit(); World.AddEntity(this); - SetTypeRef(GetType()); } public Mobile(Serial serial) @@ -351,19 +350,10 @@ public class Mobile : IHued, IComparable, ISpawnable, IObjectPropertyLis Aggressed = new List(); NextSkillTime = Core.TickCount; DamageEntries = new List(); - - SetTypeRef(GetType()); } public void SetTypeRef(Type type) { - TypeRef = World.MobileTypes.IndexOf(type); - - if (TypeRef == -1) - { - World.MobileTypes.Add(type); - TypeRef = World.MobileTypes.Count - 1; - } } public static bool DragEffects { get; set; } = true; @@ -2272,8 +2262,6 @@ public class Mobile : IHued, IComparable, ISpawnable, IObjectPropertyLis [CommandProperty(AccessLevel.Counselor)] public Serial Serial { get; } - public int TypeRef { get; private set; } - public virtual void Serialize(IGenericWriter writer) { writer.Write(33); // version diff --git a/Projects/Server/Serialization/AdhocPersistence.cs b/Projects/Server/Serialization/AdhocPersistence.cs index 086179409..8d390dc86 100644 --- a/Projects/Server/Serialization/AdhocPersistence.cs +++ b/Projects/Server/Serialization/AdhocPersistence.cs @@ -14,6 +14,8 @@ *************************************************************************/ using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.IO; using System.IO.MemoryMappedFiles; using System.Threading.Tasks; @@ -27,30 +29,53 @@ public static class AdhocPersistence * Note: The buffer may not be the same after returning from the function if more data is written * than the initial buffer can handle. */ - public static BufferWriter Serialize(Action serializer) + public static BufferWriter Serialize(Action serializer, ConcurrentQueue types) { - var saveBuffer = new BufferWriter(true); + var saveBuffer = new BufferWriter(true, types); serializer(saveBuffer); return saveBuffer; } /** * Writes a buffer to disk. This function should be called asynchronously. + * Writes the filePath for the binary data, and an accompanying SerializedTypes.db file of all possible types. */ - public static void WriteSnapshot(string filePath, Span buffer) + public static void WriteSnapshot(FileInfo file, Span buffer) { - var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory); - var file = new FileInfo(fullPath); - PathUtility.EnsureDirectory(file.DirectoryName); + var dirPath = file.DirectoryName; + PathUtility.EnsureDirectory(dirPath); - using var fs = new FileStream(fullPath, FileMode.Create, FileAccess.Write); + using var fs = new FileStream(file.FullName, FileMode.Create, FileAccess.Write); fs.Write(buffer); } - public static void SerializeAndSnapshot(string filePath, Action serializer) + /** + * Serializes to a memory buffer synchronously, then flushes to the path asynchronously. + * See WriteSnapshot for more info about how the snapshot. + */ + public static void SerializeAndSnapshot(string filePath, Action serializer, ConcurrentQueue types = null) { - var saveBuffer = Serialize(serializer); - Task.Run(() => { WriteSnapshot(filePath, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position)); }); + types ??= new ConcurrentQueue(); + var saveBuffer = Serialize(serializer, types); + Task.Run( + () => + { + var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory); + var file = new FileInfo(fullPath); + + WriteSnapshot(file, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position)); + + // TODO: Create a PooledHashSet if performance becomes an issue. + var typesSet = new HashSet(); + + // Dedupe the queue. + foreach (var type in types) + { + typesSet.Add(type); + } + + Persistence.WriteSerializedTypesSnapshot(file.DirectoryName, typesSet); + }); } public static void Deserialize(string filePath, Action deserializer) diff --git a/Projects/Server/Serialization/BinaryFileReader.cs b/Projects/Server/Serialization/BinaryFileReader.cs index eab5c7ffa..3cc5868a8 100644 --- a/Projects/Server/Serialization/BinaryFileReader.cs +++ b/Projects/Server/Serialization/BinaryFileReader.cs @@ -14,27 +14,34 @@ *************************************************************************/ using System; +using System.Collections.Generic; using System.IO; using System.Runtime.CompilerServices; using System.Text; using Server.Buffers; using Server.Collections; +using Server.Logging; using Server.Text; namespace Server; public class BinaryFileReader : IGenericReader, IDisposable { + private static readonly ILogger logger = LogFactory.GetLogger(typeof(BinaryFileReader)); + + private Dictionary _typesDb; private BinaryReader _reader; private Encoding _encoding; - public BinaryFileReader(BinaryReader br, Encoding encoding = null) + public BinaryFileReader(BinaryReader br, Dictionary typesDb = null, Encoding encoding = null) { _reader = br; _encoding = encoding ?? TextEncoding.UTF8; + _typesDb = typesDb; } - public BinaryFileReader(Stream stream, Encoding encoding = null) : this(new BinaryReader(stream), encoding) + public BinaryFileReader(Stream stream, Dictionary typesDb = null, Encoding encoding = null) + : this(new BinaryReader(stream), typesDb, encoding) { } @@ -46,23 +53,20 @@ public class BinaryFileReader : IGenericReader, IDisposable public DateTime LastSerialized { get; init; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public string ReadString(bool intern = false) - { - if (!ReadBool()) - { - return null; - } + public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null; + public string ReadStringRaw(bool intern = false) + { var length = ((IGenericReader)this).ReadEncodedInt(); if (length <= 0) { - return intern ? Utility.Intern("") : ""; + return "".Intern(); } byte[] buffer = STArrayPool.Shared.Rent(length); var str = TextEncoding.GetString(buffer.AsSpan(0, length), _encoding); STArrayPool.Shared.Return(buffer); - return intern ? Utility.Intern(str) : str; + return intern ? str.Intern() : str; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -101,6 +105,62 @@ public class BinaryFileReader : IGenericReader, IDisposable [MethodImpl(MethodImplOptions.AggressiveInlining)] public Serial ReadSerial() => (Serial)_reader.ReadUInt32(); + public Type ReadType() => + ReadByte() switch + { + 0 => null, + 1 => AssemblyHandler.FindTypeByFullName(ReadStringRaw()), // Backward compatibility + 2 => ReadTypeByHash() + }; + + public Type ReadTypeByHash() + { + var hash = ReadULong(); + var t = AssemblyHandler.FindTypeByHash(hash); + + if (t != null) + { + return t; + } + + if (_typesDb == null) + { + logger.Error( + new Exception($"The file SerializedTypes.db was not loaded. Type hash '{hash}' could not be found."), + "Invalid {Hash} at position {Position}", + hash, + Position + ); + + return null; + } + + if (!_typesDb.TryGetValue(hash, out var typeName)) + { + logger.Error( + new Exception($"Type hash '{hash}' is not present in the serialized types database."), + "Invalid type hash {Hash} at position {Position}", + hash, + Position + ); + + return null; + } + + t = AssemblyHandler.FindTypeByFullName(typeName, false); + + if (t == null) + { + logger.Error( + new Exception($"Type '{typeName}' was not found."), + "Type {Type} was not found.", + typeName + ); + } + + return t; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Read(Span buffer) => _reader.Read(buffer); diff --git a/Projects/Server/Serialization/BinaryFileWriter.cs b/Projects/Server/Serialization/BinaryFileWriter.cs index ceb7447c1..3febd65c1 100644 --- a/Projects/Server/Serialization/BinaryFileWriter.cs +++ b/Projects/Server/Serialization/BinaryFileWriter.cs @@ -14,6 +14,7 @@ *************************************************************************/ using System; +using System.Collections.Concurrent; using System.IO; namespace Server; @@ -23,11 +24,11 @@ public class BinaryFileWriter : BufferWriter, IDisposable private readonly Stream _file; private long _position; - public BinaryFileWriter(string filename, bool prefixStr) : - this(new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None), prefixStr) + public BinaryFileWriter(string filename, bool prefixStr, ConcurrentQueue types = null) : + this(new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None), prefixStr, types) {} - public BinaryFileWriter(Stream stream, bool prefixStr) : base(prefixStr) + public BinaryFileWriter(Stream stream, bool prefixStr, ConcurrentQueue types = null) : base(prefixStr, types) { _file = stream; _position = _file.Position; diff --git a/Projects/Server/Serialization/BufferReader.cs b/Projects/Server/Serialization/BufferReader.cs index 1a1f151af..eece3f03a 100644 --- a/Projects/Server/Serialization/BufferReader.cs +++ b/Projects/Server/Serialization/BufferReader.cs @@ -15,30 +15,40 @@ using System; using System.Buffers.Binary; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Runtime.CompilerServices; using System.Text; using Server.Collections; +using Server.Logging; using Server.Text; namespace Server; public class BufferReader : IGenericReader { + private static readonly ILogger logger = LogFactory.GetLogger(typeof(BufferReader)); + + private Dictionary _typesDb; private Encoding _encoding; private byte[] _buffer; private int _position; public long Position => _position; - public BufferReader(byte[] buffer, Encoding encoding = null) + public BufferReader(byte[] buffer, Dictionary typesDb = null, Encoding encoding = null) { _buffer = buffer; _encoding = encoding ?? TextEncoding.UTF8; + _typesDb = typesDb; } - public BufferReader(byte[] buffer, DateTime lastSerialized) : this(buffer) => LastSerialized = lastSerialized; + public BufferReader(byte[] buffer, DateTime lastSerialized, Dictionary typesDb = null) : this(buffer) + { + LastSerialized = lastSerialized; + _typesDb = typesDb; + } public void Reset(byte[] newBuffer, out byte[] oldBuffer) { @@ -49,22 +59,20 @@ public class BufferReader : IGenericReader public DateTime LastSerialized { get; init; } - public string ReadString(bool intern = false) - { - if (!ReadBool()) - { - return null; - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null; + public string ReadStringRaw(bool intern = false) + { var length = ((IGenericReader)this).ReadEncodedInt(); if (length <= 0) { - return intern ? Utility.Intern("") : ""; + return "".Intern(); } var str = TextEncoding.GetString(_buffer.AsSpan(_position, length), _encoding); _position += length; - return intern ? Utility.Intern(str) : str; + return intern ? str.Intern() : str; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -143,6 +151,62 @@ public class BufferReader : IGenericReader [MethodImpl(MethodImplOptions.AggressiveInlining)] public Serial ReadSerial() => (Serial)ReadUInt(); + public Type ReadType() => + ReadByte() switch + { + 0 => null, + 1 => AssemblyHandler.FindTypeByFullName(ReadStringRaw()), // Backward compatibility + 2 => ReadTypeByHash() + }; + + public Type ReadTypeByHash() + { + var hash = ReadULong(); + var t = AssemblyHandler.FindTypeByHash(hash); + + if (t != null) + { + return t; + } + + if (_typesDb == null) + { + logger.Error( + new Exception($"The file SerializedTypes.db was not loaded. Type hash '{hash}' could not be found."), + "Invalid {Hash} at position {Position}", + hash, + Position + ); + + return null; + } + + if (!_typesDb.TryGetValue(hash, out var typeName)) + { + logger.Error( + new Exception($"Type hash '{hash}' is not present in the serialized types database."), + "Invalid type hash {Hash} at position {Position}", + hash, + Position + ); + + return null; + } + + t = AssemblyHandler.FindTypeByFullName(typeName, false); + + if (t == null) + { + logger.Error( + new Exception($"Type '{typeName}' was not found."), + "Type {Type} was not found.", + typeName + ); + } + + return t; + } + public int Read(Span buffer) { var length = buffer.Length; diff --git a/Projects/Server/Serialization/BufferWriter.cs b/Projects/Server/Serialization/BufferWriter.cs index 325b33717..a3090c564 100644 --- a/Projects/Server/Serialization/BufferWriter.cs +++ b/Projects/Server/Serialization/BufferWriter.cs @@ -2,7 +2,7 @@ * ModernUO * * Copyright 2019-2022 - ModernUO Development Team * * Email: hi@modernuo.com * - * File: BufferedFileWriter.cs * + * File: BufferWriter.cs * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * @@ -14,6 +14,7 @@ *************************************************************************/ using System; +using System.Collections.Concurrent; using System.Diagnostics; using System.IO; using System.Runtime.CompilerServices; @@ -25,8 +26,9 @@ namespace Server; public class BufferWriter : IGenericWriter { - private readonly Encoding m_Encoding; - private readonly bool m_PrefixStrings; + private ConcurrentQueue _types; + private Encoding _encoding; + private bool _prefixStrings; private long _bytesWritten; private long _index; @@ -53,22 +55,24 @@ public class BufferWriter : IGenericWriter private byte[] _buffer; - public BufferWriter(byte[] buffer, bool prefixStr) + public BufferWriter(byte[] buffer, bool prefixStr, ConcurrentQueue types = null) { - m_PrefixStrings = prefixStr; - m_Encoding = TextEncoding.UTF8; + _prefixStrings = prefixStr; + _encoding = TextEncoding.UTF8; _buffer = buffer; + _types = types; } - public BufferWriter(bool prefixStr) : this(0, prefixStr) + public BufferWriter(bool prefixStr, ConcurrentQueue types = null) : this(0, prefixStr, types) { } - public BufferWriter(int count, bool prefixStr) + public BufferWriter(int count, bool prefixStr, ConcurrentQueue types = null) { - m_PrefixStrings = prefixStr; - m_Encoding = TextEncoding.UTF8; + _prefixStrings = prefixStr; + _encoding = TextEncoding.UTF8; _buffer = GC.AllocateUninitializedArray(count < 1 ? BufferSize : count); + _types = types; } public virtual long Position => Index; @@ -135,6 +139,7 @@ public class BufferWriter : IGenericWriter } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(BitArray bitArray) { var byteLength = BitArray.GetByteArrayLengthFromBitLength(bitArray.Length); @@ -145,6 +150,7 @@ public class BufferWriter : IGenericWriter Index += byteLength; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public virtual long Seek(long offset, SeekOrigin origin) { Debug.Assert( @@ -168,9 +174,10 @@ public class BufferWriter : IGenericWriter }); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(string value) { - if (m_PrefixStrings) + if (_prefixStrings) { if (value == null) { @@ -188,6 +195,7 @@ public class BufferWriter : IGenericWriter } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(long value) { FlushIfNeeded(8); @@ -202,6 +210,7 @@ public class BufferWriter : IGenericWriter _buffer[Index++] = (byte)(value >> 56); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(ulong value) { FlushIfNeeded(8); @@ -216,6 +225,7 @@ public class BufferWriter : IGenericWriter _buffer[Index++] = (byte)(value >> 56); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(int value) { FlushIfNeeded(4); @@ -226,6 +236,7 @@ public class BufferWriter : IGenericWriter _buffer[Index++] = (byte)(value >> 24); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(uint value) { FlushIfNeeded(4); @@ -236,6 +247,7 @@ public class BufferWriter : IGenericWriter _buffer[Index++] = (byte)(value >> 24); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(short value) { FlushIfNeeded(2); @@ -244,6 +256,7 @@ public class BufferWriter : IGenericWriter _buffer[Index++] = (byte)(value >> 8); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(ushort value) { FlushIfNeeded(2); @@ -252,6 +265,7 @@ public class BufferWriter : IGenericWriter _buffer[Index++] = (byte)(value >> 8); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public unsafe void Write(double value) { FlushIfNeeded(8); @@ -264,6 +278,7 @@ public class BufferWriter : IGenericWriter Index += 8; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public unsafe void Write(float value) { FlushIfNeeded(4); @@ -300,9 +315,25 @@ public class BufferWriter : IGenericWriter [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(Serial serial) => Write(serial.Value); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Write(Type type) + { + if (type == null) + { + Write((byte)0); + } + else + { + Write((byte)0x2); // xxHash3 64bit + Write(AssemblyHandler.GetTypeHash(type)); + _types?.Enqueue(type); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void InternalWriteString(string value) { - var remaining = m_Encoding.GetByteCount(value); + var remaining = _encoding.GetByteCount(value); ((IGenericWriter)this).WriteEncodedInt(remaining); @@ -313,14 +344,14 @@ public class BufferWriter : IGenericWriter // It is much faster to encode to stack buffer, then copy to the real buffer Span span = stackalloc byte[Math.Min(BufferSize, 256)]; - var maxChars = span.Length / m_Encoding.GetMaxByteCount(1); + var maxChars = span.Length / _encoding.GetMaxByteCount(1); var charsLeft = value.Length; var current = 0; while (charsLeft > 0) { var charCount = Math.Min(charsLeft, maxChars); - var bytesWritten = m_Encoding.GetBytes(value.AsSpan(current, charCount), span); + var bytesWritten = _encoding.GetBytes(value.AsSpan(current, charCount), span); remaining -= bytesWritten; charsLeft -= charCount; current += charCount; diff --git a/Projects/Server/Serialization/GenericPersistence.cs b/Projects/Server/Serialization/GenericPersistence.cs index 355f21433..fa47736cd 100644 --- a/Projects/Server/Serialization/GenericPersistence.cs +++ b/Projects/Server/Serialization/GenericPersistence.cs @@ -14,6 +14,8 @@ *************************************************************************/ using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.IO; namespace Server; @@ -31,22 +33,22 @@ public static class GenericPersistence void Serialize() { - saveBuffer ??= new BufferWriter(true); + saveBuffer ??= new BufferWriter(true, World.SerializedTypes); saveBuffer.Seek(0, SeekOrigin.Begin); serializer(saveBuffer); } - void WriterSnapshot(string savePath) + void WriteSnapshot(string savePath) { string binPath = Path.Combine(savePath, name, $"{name}.bin"); var buffer = saveBuffer!.Buffer.AsSpan(0, (int)saveBuffer.Position); - AdhocPersistence.WriteSnapshot(binPath, buffer); + AdhocPersistence.WriteSnapshot(new FileInfo(binPath), buffer); } - void Deserialize(string savePath) => + void Deserialize(string savePath, Dictionary typesDb) => AdhocPersistence.Deserialize(Path.Combine(savePath, name, $"{name}.bin"), deserializer); - Persistence.Register(name, Serialize, WriterSnapshot, Deserialize, priority); + Persistence.Register(name, Serialize, WriteSnapshot, Deserialize, priority); } } diff --git a/Projects/Server/Serialization/IGenericReader.cs b/Projects/Server/Serialization/IGenericReader.cs index 837c6e212..f5b29f611 100644 --- a/Projects/Server/Serialization/IGenericReader.cs +++ b/Projects/Server/Serialization/IGenericReader.cs @@ -38,6 +38,7 @@ public interface IGenericReader sbyte ReadSByte(); bool ReadBool(); Serial ReadSerial(); + Type ReadType(); DateTime ReadDateTime() => new(ReadLong(), DateTimeKind.Utc); TimeSpan ReadTimeSpan() => new(ReadLong()); diff --git a/Projects/Server/Serialization/IGenericWriter.cs b/Projects/Server/Serialization/IGenericWriter.cs index 97aa76ca5..a7dd75a1a 100644 --- a/Projects/Server/Serialization/IGenericWriter.cs +++ b/Projects/Server/Serialization/IGenericWriter.cs @@ -37,6 +37,7 @@ public interface IGenericWriter void Write(sbyte value); void Write(bool value); void Write(Serial serial); + void Write(Type type); void Write(DateTime value) { diff --git a/Projects/Server/Serialization/ISerializable.cs b/Projects/Server/Serialization/ISerializable.cs index f49bd50f5..9452f87d3 100644 --- a/Projects/Server/Serialization/ISerializable.cs +++ b/Projects/Server/Serialization/ISerializable.cs @@ -14,6 +14,7 @@ *************************************************************************/ using System; +using System.Collections.Concurrent; using System.IO; namespace Server; @@ -28,7 +29,6 @@ public interface ISerializable long SavePosition { get; protected internal set; } BufferWriter SaveBuffer { get; protected internal set; } - int TypeRef { get; } Serial Serial { get; } // Executed on every entity, before it's serialized. @@ -39,11 +39,9 @@ public interface ISerializable void Delete(); bool Deleted { get; } - void SetTypeRef(Type type); - - public void InitializeSaveBuffer(byte[] buffer) + public void InitializeSaveBuffer(byte[] buffer, ConcurrentQueue types) { - SaveBuffer = new BufferWriter(buffer, true); + SaveBuffer = new BufferWriter(buffer, true, types); if (World.DirtyTrackingEnabled) { SavePosition = SaveBuffer.Position; @@ -54,9 +52,9 @@ public interface ISerializable } } - public void Serialize() + public void Serialize(ConcurrentQueue types) { - SaveBuffer ??= new BufferWriter(true); + SaveBuffer ??= new BufferWriter(true, types); BeforeSerialize(); diff --git a/Projects/Server/Serialization/Persistence.cs b/Projects/Server/Serialization/Persistence.cs index 048b087ba..e27b3fba4 100644 --- a/Projects/Server/Serialization/Persistence.cs +++ b/Projects/Server/Serialization/Persistence.cs @@ -14,13 +14,14 @@ *************************************************************************/ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Threading.Tasks; namespace Server; -public class Persistence +public static class Persistence { public const int DefaultPriority = 100; @@ -30,7 +31,7 @@ public class Persistence string name, Action serializer, Action snapshotWriter, - Action deserializer, + Action> deserializer, int priority = DefaultPriority ) { @@ -50,24 +51,79 @@ public class Persistence public static void Load(string path) { + var typesDb = LoadTypes(path); + // This should probably not be parallel since Mobiles must be loaded before Items foreach (var entry in _registry) { - entry.Deserialize(path); + entry.Deserialize(path, typesDb); } } + private static Dictionary LoadTypes(string path) + { + var db = new Dictionary(); + + string tdbPath = Path.Combine(path, "SerializedTypes.db"); + if (!File.Exists(tdbPath)) + { + return db; + } + + using FileStream tdb = new FileStream(tdbPath, FileMode.Open, FileAccess.Read, FileShare.Read); + BinaryReader tdbReader = new BinaryReader(tdb); + + var version = tdbReader.ReadInt32(); + var count = tdbReader.ReadInt32(); + + for (var i = 0; i < count; ++i) + { + var hash = tdbReader.ReadUInt64(); + var typeName = tdbReader.ReadString(); + db[hash] = typeName; + } + + return db; + } + public static void Serialize() { Parallel.ForEach(_registry, entry => entry.Serialize()); } - public static void WriteSnapshot(string path) + public static void WriteSnapshot(string path, ConcurrentQueue types) { foreach (var entry in _registry) { entry.WriteSnapshot(path); } + + // Dedupe the queue. + foreach (var type in types) + { + _typesSet.Add(type); + } + + WriteSerializedTypesSnapshot(path, _typesSet); + _typesSet.Clear(); + } + + private static HashSet _typesSet = new(); + + public static void WriteSerializedTypesSnapshot(string path, HashSet types) + { + string tdbPath = Path.Combine(path, "SerializedTypes.db"); + using var tdb = new BinaryFileWriter(tdbPath, false); + + tdb.Write(0); // version + tdb.Write(types.Count); + + foreach (var type in types) + { + var fullName = type.FullName; + tdb.Write(HashUtility.ComputeHash64(fullName)); + tdb.Write(fullName); + } } public record RegistryEntry @@ -76,7 +132,7 @@ public class Persistence public int Priority { get; init; } public Action Serialize { get; init; } // Serializing to memory buffers public Action WriteSnapshot { get; init; } - public Action Deserialize { get; init; } + public Action> Deserialize { get; init; } } internal class RegistryEntryComparer : IComparer diff --git a/Projects/Server/Server.csproj b/Projects/Server/Server.csproj index db2d421d6..53ca56095 100755 --- a/Projects/Server/Server.csproj +++ b/Projects/Server/Server.csproj @@ -38,6 +38,7 @@ + diff --git a/Projects/Server/Utilities/HashUtility.cs b/Projects/Server/Utilities/HashUtility.cs new file mode 100644 index 000000000..e757c5140 --- /dev/null +++ b/Projects/Server/Utilities/HashUtility.cs @@ -0,0 +1,48 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2022 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: HashUtility.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.Runtime.CompilerServices; +using Standart.Hash.xxHash; + +namespace Server; + +/// +/// Represents supported non-cryptographic fast hash algorithms. +/// +public enum FastHashAlgorithm +{ + None, // Used for collisions where full-data is serialized instead + XXHash3_64, // xxHash3 64bit +} + +public static class HashUtility +{ + // *************** DO NOT CHANGE THIS NUMBER **************** + // * Computed hashes might be serialized against this seed! * + // ********************************************************** + private const ulong xxHash3Seed = 9609125370673258709ul; // Randomly generated 64-bit prime number + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ulong ComputeHash64(string? data, FastHashAlgorithm algorithm = FastHashAlgorithm.XXHash3_64) => + algorithm switch + { + FastHashAlgorithm.XXHash3_64 => ComputeXXHash3_64(data), + _ => throw new NotSupportedException($"Hash {algorithm} is not supported.") + }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ulong ComputeXXHash3_64(string? data) => data == null ? 0 : xxHash3.ComputeHash(data, xxHash3Seed); +} diff --git a/Projects/Server/Utilities/Utility.cs b/Projects/Server/Utilities/Utility.cs index 0ef9c036c..21b986818 100644 --- a/Projects/Server/Utilities/Utility.cs +++ b/Projects/Server/Utilities/Utility.cs @@ -120,8 +120,10 @@ public static class Utility sb.Append(value); } - public static string Intern(string str) => str?.Length > 0 ? string.Intern(str) : str; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static string Intern(this string str) => str?.Length > 0 ? string.Intern(str) : str; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Intern(ref string str) { str = Intern(str); diff --git a/Projects/Server/World/EntityPersistence.cs b/Projects/Server/World/EntityPersistence.cs index 8586ae7dd..69d9b37fa 100644 --- a/Projects/Server/World/EntityPersistence.cs +++ b/Projects/Server/World/EntityPersistence.cs @@ -14,6 +14,7 @@ *************************************************************************/ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.IO.MemoryMappedFiles; @@ -25,13 +26,11 @@ namespace Server; public static class EntityPersistence { - private const int _idxVersion = 1; - public static void WriteEntities( IIndexInfo indexInfo, Dictionary entities, - List types, string savePath, + ConcurrentQueue types, out Dictionary counts ) where T : class, ISerializable { @@ -44,20 +43,19 @@ public static class EntityPersistence PathUtility.EnsureDirectory(path); string idxPath = Path.Combine(path, $"{typeName}.idx"); - string tdbPath = Path.Combine(path, $"{typeName}.tdb"); string binPath = Path.Combine(path, $"{typeName}.bin"); - using var idx = new BinaryFileWriter(idxPath, false); - using var tdb = new BinaryFileWriter(tdbPath, false); - using var bin = new BinaryFileWriter(binPath, true); + using var idx = new BinaryFileWriter(idxPath, false, types); + using var bin = new BinaryFileWriter(binPath, true, types); - idx.Write(1); // Version + idx.Write(2); // Version idx.Write(entities.Count); foreach (var e in entities.Values) { long start = bin.Position; - idx.Write(e.TypeRef); + var t = e.GetType(); + idx.Write(t); idx.Write(e.Serial); idx.Write(e.Created.Ticks); idx.Write(e.LastSerialized.Ticks); @@ -73,12 +71,6 @@ public static class EntityPersistence counts[type] = (counts.TryGetValue(type, out var count) ? count : 0) + 1; } } - - tdb.Write(types.Count); - for (int i = 0; i < types.Count; ++i) - { - tdb.Write(types[i].FullName); - } } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -90,6 +82,7 @@ public static class EntityPersistence public static Dictionary LoadIndex( string path, IIndexInfo indexInfo, + Dictionary serializedTypes, out List> entities ) where T : class, ISerializable { @@ -99,11 +92,10 @@ public static class EntityPersistence var indexType = indexInfo.TypeName; string indexPath = Path.Combine(path, indexType, $"{indexType}.idx"); - string typesPath = Path.Combine(path, indexType, $"{indexType}.tdb"); entities = new List>(); - if (!File.Exists(indexPath) || !File.Exists(typesPath)) + if (!File.Exists(indexPath)) { return map; } @@ -111,58 +103,77 @@ public static class EntityPersistence using FileStream idx = new FileStream(indexPath, FileMode.Open, FileAccess.Read, FileShare.Read); BinaryReader idxReader = new BinaryReader(idx); - using FileStream tdb = new FileStream(typesPath, FileMode.Open, FileAccess.Read, FileShare.Read); - BinaryReader tdbReader = new BinaryReader(tdb); - - List> types = ReadTypes(tdbReader); - - int count; var version = idxReader.ReadInt32(); + int count = idxReader.ReadInt32(); - // Handle non-versioned (version 0). - if (version > _idxVersion || idx.Length - 4 - version * 20 == 0) + var ctorArguments = new[] { typeof(I) }; + List types; + + string typesPath = Path.Combine(path, indexType, $"{indexType}.tdb"); + if (File.Exists(typesPath)) { - count = version; - version = 0; + using FileStream tdb = new FileStream(typesPath, FileMode.Open, FileAccess.Read, FileShare.Read); + BinaryReader tdbReader = new BinaryReader(tdb); + types = ReadTypes(tdbReader, ctorArguments); + tdbReader.Close(); } else { - count = idxReader.ReadInt32(); + types = null; + } + + // We must have a typeDb from SerializedTypes.db, or a tdb file + if (serializedTypes == null && types == null) + { + return map; } var now = DateTime.UtcNow; for (int i = 0; i < count; ++i) { - var typeID = idxReader.ReadInt32(); + ConstructorInfo ctor; + if (version >= 2) + { + var flag = idxReader.ReadByte(); + if (flag != 2) + { + throw new Exception($"Invalid type flag, expected 2 but received {flag}."); + } + + var hash = idxReader.ReadUInt64(); + serializedTypes!.TryGetValue(hash, out var typeName); + ctor = GetConstructorFor(typeName, AssemblyHandler.FindTypeByHash(hash), ctorArguments); + } + else + { + ctor = types?[idxReader.ReadInt32()]; + } + var serial = idxReader.ReadUInt32(); var created = version == 0 ? now : new DateTime(idxReader.ReadInt64(), DateTimeKind.Utc); var lastSerialized = version == 0 ? DateTime.MinValue : new DateTime(idxReader.ReadInt64(), DateTimeKind.Utc); var pos = idxReader.ReadInt64(); var length = idxReader.ReadInt32(); - Tuple objs = types[typeID]; - - if (objs == null) + if (ctor == null) { continue; } - ConstructorInfo ctor = objs.Item1; I indexer = indexInfo.CreateIndex(serial); ctorArgs[0] = indexer; - if (ctor.Invoke(ctorArgs) is T t) + if (ctor.Invoke(ctorArgs) is T entity) { - t.Created = created; - t.LastSerialized = lastSerialized; - entities.Add(new EntitySpan(t, typeID, pos, length)); - map[indexer] = t; + entity.Created = created; + entity.LastSerialized = lastSerialized; + entities.Add(new EntitySpan(entity, pos, length)); + map[indexer] = entity; } } - tdbReader.Close(); idxReader.Close(); return map; @@ -171,6 +182,7 @@ public static class EntityPersistence public static void LoadData( string path, IIndexInfo indexInfo, + Dictionary serializedTypes, List> entities ) where T : class, ISerializable { @@ -211,7 +223,7 @@ public static class EntityPersistence var buffer = GC.AllocateUninitializedArray(entry.Length); if (br == null) { - br = new BufferReader(buffer, t.LastSerialized); + br = new BufferReader(buffer, t.LastSerialized, serializedTypes); } else { @@ -236,7 +248,7 @@ public static class EntityPersistence if (error == null) { - t.InitializeSaveBuffer(buffer); + t.InitializeSaveBuffer(buffer, World.SerializedTypes); } else { @@ -265,50 +277,52 @@ public static class EntityPersistence } } - private static List> ReadTypes(BinaryReader tdbReader) + private static ConstructorInfo GetConstructorFor(string typeName, Type t, Type[] constructorTypes) { - var constructorTypes = new[] { typeof(I) }; + if (t?.IsAbstract != false) + { + Console.WriteLine("failed"); + var issue = t?.IsAbstract == true ? "marked abstract" : "not found"; + + Console.WriteLine($"Error: Type '{typeName}' was {issue}. Delete all of those types? (y/n)"); + + if (Console.ReadKey(true).Key == ConsoleKey.Y) + { + Console.WriteLine("Loading..."); + return null; + } + + Console.WriteLine("Types will not be deleted. An exception will be thrown."); + + throw new Exception($"Bad type '{typeName}'"); + } + + var ctor = t.GetConstructor(constructorTypes); + + if (ctor == null) + { + throw new Exception($"Type '{t}' does not have a serialization constructor"); + } + + return ctor; + } + + /** + * Legacy ReadTypes for backward compatibility with old saves that still have a tdb file + */ + private static List ReadTypes(BinaryReader tdbReader, Type[] ctorArguments) + { var count = tdbReader.ReadInt32(); - var types = new List>(count); + var types = new List(count); for (var i = 0; i < count; ++i) { var typeName = tdbReader.ReadString(); - var t = AssemblyHandler.FindTypeByFullName(typeName, false); - - if (t?.IsAbstract != false) - { - Console.WriteLine("failed"); - - var issue = t?.IsAbstract == true ? "marked abstract" : "not found"; - - Console.WriteLine($"Error: Type '{typeName}' was {issue}. Delete all of those types? (y/n)"); - - if (Console.ReadKey(true).Key == ConsoleKey.Y) - { - types.Add(null); - Console.WriteLine("Loading..."); - continue; - } - - Console.WriteLine("Types will not be deleted. An exception will be thrown."); - - throw new Exception($"Bad type '{typeName}'"); - } - - var ctor = t.GetConstructor(constructorTypes); - - if (ctor != null) - { - types.Add(new Tuple(ctor, typeName)); - } - else - { - throw new Exception($"Type '{t}' does not have a serialization constructor"); - } + var ctor = GetConstructorFor(typeName, t, ctorArguments); + types.Add(ctor); } return types; diff --git a/Projects/Server/World/EntitySpan.cs b/Projects/Server/World/EntitySpan.cs index 16f8670df..c01f06514 100644 --- a/Projects/Server/World/EntitySpan.cs +++ b/Projects/Server/World/EntitySpan.cs @@ -19,16 +19,13 @@ public struct EntitySpan where T : ISerializable { public T Entity { get; } - public int TypeID { get; } - public long Position { get; } public int Length { get; } - public EntitySpan(T entity, int typeID, long position, int length) + public EntitySpan(T entity, long position, int length) { Entity = entity; - TypeID = typeID; Position = position; Length = length; } diff --git a/Projects/Server/World/World.cs b/Projects/Server/World/World.cs index 818a67883..2c0893596 100644 --- a/Projects/Server/World/World.cs +++ b/Projects/Server/World/World.cs @@ -38,12 +38,12 @@ public enum WorldState public static class World { - private static readonly ILogger logger = LogFactory.GetLogger(typeof(World)); + private static ILogger logger = LogFactory.GetLogger(typeof(World)); - private static readonly ManualResetEvent m_DiskWriteHandle = new(true); - private static readonly Dictionary _pendingAdd = new(); - private static readonly Dictionary _pendingDelete = new(); - private static readonly ConcurrentQueue _decayQueue = new(); + private static ManualResetEvent m_DiskWriteHandle = new(true); + private static Dictionary _pendingAdd = new(); + private static Dictionary _pendingDelete = new(); + private static ConcurrentQueue _decayQueue = new(); private static string _tempSavePath; // Path to the temporary folder for the save private static bool _enableSaveStats; @@ -125,10 +125,6 @@ public static class World private static void OutOfMemory(string message) => throw new OutOfMemoryException(message); - internal static List ItemTypes { get; } = new(); - internal static List MobileTypes { get; } = new(); - internal static List GuildTypes { get; } = new(); - public static string SavePath { get; private set; } public static WorldState WorldState { get; private set; } @@ -232,15 +228,15 @@ public static class World public static void Broadcast(int hue, bool ascii, string format, params object[] args) => Broadcast(hue, ascii, string.Format(format, args)); - internal static void LoadEntities(string basePath) + internal static void LoadEntities(string basePath, Dictionary typesDb) { IIndexInfo itemIndexInfo = new EntityTypeIndex("Items"); IIndexInfo mobileIndexInfo = new EntityTypeIndex("Mobiles"); IIndexInfo guildIndexInfo = new EntityTypeIndex("Guilds"); - Mobiles = EntityPersistence.LoadIndex(basePath, mobileIndexInfo, out List> mobiles); - Items = EntityPersistence.LoadIndex(basePath, itemIndexInfo, out List> items); - Guilds = EntityPersistence.LoadIndex(basePath, guildIndexInfo, out List> guilds); + Mobiles = EntityPersistence.LoadIndex(basePath, mobileIndexInfo, typesDb, out List> mobiles); + Items = EntityPersistence.LoadIndex(basePath, itemIndexInfo, typesDb, out List> items); + Guilds = EntityPersistence.LoadIndex(basePath, guildIndexInfo, typesDb, out List> guilds); if (Mobiles.Count > 0) { @@ -257,9 +253,9 @@ public static class World _lastGuild = Guilds.Keys.Max(); } - EntityPersistence.LoadData(basePath, mobileIndexInfo, mobiles); - EntityPersistence.LoadData(basePath, itemIndexInfo, items); - EntityPersistence.LoadData(basePath, guildIndexInfo, guilds); + EntityPersistence.LoadData(basePath, mobileIndexInfo, typesDb, mobiles); + EntityPersistence.LoadData(basePath, itemIndexInfo, typesDb, items); + EntityPersistence.LoadData(basePath, guildIndexInfo, typesDb, guilds); } public static void Load() @@ -392,9 +388,9 @@ public static class World IIndexInfo mobileIndexInfo = new EntityTypeIndex("Mobiles"); IIndexInfo guildIndexInfo = new EntityTypeIndex("Guilds"); - EntityPersistence.WriteEntities(mobileIndexInfo, Mobiles, MobileTypes, basePath, out var mobileCounts); - EntityPersistence.WriteEntities(itemIndexInfo, Items, ItemTypes, basePath, out var itemCounts); - EntityPersistence.WriteEntities(guildIndexInfo, Guilds, GuildTypes, basePath, out var guildCounts); + EntityPersistence.WriteEntities(mobileIndexInfo, Mobiles, basePath, SerializedTypes, out var mobileCounts); + EntityPersistence.WriteEntities(itemIndexInfo, Items, basePath, SerializedTypes, out var itemCounts); + EntityPersistence.WriteEntities(guildIndexInfo, Guilds, basePath, SerializedTypes, out var guildCounts); if (_enableSaveStats) { @@ -413,7 +409,7 @@ public static class World var watch = Stopwatch.StartNew(); logger.Information("Writing world save snapshot"); - Persistence.WriteSnapshot(tempPath); + Persistence.WriteSnapshot(tempPath, SerializedTypes); watch.Stop(); @@ -444,6 +440,9 @@ public static class World } } + // Clear types + SerializedTypes.Clear(); + m_DiskWriteHandle.Set(); Core.LoopContext.Post(FinishWorldSave); @@ -463,6 +462,39 @@ public static class World private static DateTime _serializationStart; + /** + * Duplicates can be weeded out asynchronously while flushing + * If performance becomes a problem, we need to build a dual mode concurrent array. + * + ****************************************************** Proposal ****************************************************** + * The structure is initialized with a large capacity to avoid unnecessary resizing. + * Write Mode: + * - Multiple threads can add a single, or a range of elements concurrently. + * - Elements can be Peeked, but there are no guarantees. + * - To resize the internal array, replaced it with the next size up from an array pool. + * - The structure cannot be cleared in this mode. + * + * Read Mode: + * - The array can be read from multiple threads using a ref struct enumerator. + * - Elements cannot be added or reassigned. + * - Cleared by replacing the internal array with another one from the pool. + * - Note: Upon clearing, the existing array is not sent back to the pool until there are zero enumerators. + * + * Enumeration: + * - Multiple threads can enumerate while in read mode. The enumerator will Interlocked.Increment a read counter. + * - Upon dispose of the enumerator, the read counter will be lowered with an Interlocked.Decrement + * - When the read counter reaches 0, if there is a cleared array, the array is sent back to the pool zeroed. + * + * Notes: + * - Elements can never be removed. + * + * How is this different from ConcurrentQueue? + * The functionality is very similar, except the constraints allow the implementation to be done without locks. + * Since this implementation uses pooled arrays, allocations will approach zero over time. + ********************************************************************************************************************** + */ + public static ConcurrentQueue SerializedTypes { get; } = new(); + internal static void SaveEntities() { _serializationStart = DateTime.UtcNow; @@ -479,7 +511,7 @@ public static class World EnqueueForDecay(item); } - entity.Serialize(); + entity.Serialize(SerializedTypes); } public static void Save() @@ -632,7 +664,10 @@ public static class World } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void AddGuild(BaseGuild guild) => Guilds[guild.Serial] = guild; + public static void AddGuild(BaseGuild guild) + { + Guilds[guild.Serial] = guild; + } public static void RemoveEntity(T entity) where T : class, IEntity { diff --git a/Projects/UOContent/Accounting/Account.cs b/Projects/UOContent/Accounting/Account.cs index 34ebc5ddb..731974cfc 100644 --- a/Projects/UOContent/Accounting/Account.cs +++ b/Projects/UOContent/Accounting/Account.cs @@ -15,6 +15,10 @@ namespace Server.Accounting [SerializationGenerator(4)] public partial class Account : IAccount, IComparable, ISerializable { + public void SetTypeRef(Type type) + { + } + public static readonly TimeSpan YoungDuration = TimeSpan.FromHours(40.0); public static readonly TimeSpan InactiveDuration = TimeSpan.FromDays(180.0); public static readonly TimeSpan EmptyInactiveDuration = TimeSpan.FromDays(30.0); @@ -145,7 +149,6 @@ namespace Server.Accounting public Account(XmlElement node) { Serial = Accounts.NewAccount; - SetTypeRef(GetType()); _username = Utility.GetText(node["username"], "empty"); @@ -218,17 +221,6 @@ namespace Server.Accounting this.MarkDirty(); } - public void SetTypeRef(Type type) - { - TypeRef = Accounts.Types.IndexOf(type); - - if (TypeRef == -1) - { - Accounts.Types.Add(type); - TypeRef = Accounts.Types.Count - 1; - } - } - /// /// Object detailing information about the hardware of the last person to log into this account /// @@ -305,8 +297,6 @@ namespace Server.Accounting [CommandProperty(AccessLevel.GameMaster)] DateTime ISerializable.LastSerialized { get; set; } = Core.Now; - public int TypeRef { get; private set; } - public Serial Serial { get; set; } public void BeforeSerialize() diff --git a/Projects/UOContent/Accounting/Accounts.cs b/Projects/UOContent/Accounting/Accounts.cs index 0d81363b9..b71dfad7e 100644 --- a/Projects/UOContent/Accounting/Accounts.cs +++ b/Projects/UOContent/Accounting/Accounts.cs @@ -14,7 +14,6 @@ namespace Server.Accounting private static readonly Dictionary _accountsByName = new(32, StringComparer.OrdinalIgnoreCase); private static Dictionary _accountsById = new(32); private static Serial _lastAccount; - internal static List Types { get; } = new(); private static void OutOfMemory(string message) => throw new OutOfMemoryException(message); @@ -44,13 +43,18 @@ namespace Server.Accounting public static void Configure() => Persistence.Register("Accounts", Serialize, WriteSnapshot, Deserialize); - internal static void Serialize() => - EntityPersistence.SaveEntities(_accountsById.Values, account => ((ISerializable)account).Serialize()); + internal static void Serialize() + { + EntityPersistence.SaveEntities( + _accountsById.Values, + account => ((ISerializable)account).Serialize(World.SerializedTypes) + ); + } internal static void WriteSnapshot(string basePath) { IIndexInfo indexInfo = new EntityTypeIndex("Accounts"); - EntityPersistence.WriteEntities(indexInfo, _accountsById, Types, basePath, out _); + EntityPersistence.WriteEntities(indexInfo, _accountsById, basePath,World.SerializedTypes, out _); } public static IEnumerable GetAccounts() => _accountsByName.Values; @@ -73,7 +77,7 @@ namespace Server.Accounting _accountsById.Remove(a.Serial); } - internal static void Deserialize(string path) + internal static void Deserialize(string path, Dictionary typesDb) { var filePath = Path.Combine(path, "Accounts", "accounts.xml"); @@ -86,14 +90,14 @@ namespace Server.Accounting IIndexInfo indexInfo = new EntityTypeIndex("Accounts"); - _accountsById = EntityPersistence.LoadIndex(path, indexInfo, out List> accounts); + _accountsById = EntityPersistence.LoadIndex(path, indexInfo, typesDb, out List> accounts); if (_accountsById.Count > 0) { _lastAccount = _accountsById.Keys.Max(); } - EntityPersistence.LoadData(path, indexInfo, accounts); + EntityPersistence.LoadData(path, indexInfo, typesDb, accounts); foreach (var a in _accountsById.Values) { diff --git a/Projects/UOContent/Items/Armor/Cloth/GargishClothKiltType2.cs b/Projects/UOContent/Items/Armor/Cloth/GargishClothKiltType2.cs index 45c836fb4..8f9e4a486 100644 --- a/Projects/UOContent/Items/Armor/Cloth/GargishClothKiltType2.cs +++ b/Projects/UOContent/Items/Armor/Cloth/GargishClothKiltType2.cs @@ -3,7 +3,6 @@ using ModernUO.Serialization; namespace Server.Items { [SerializationGenerator(0)] - [TypeAlias("Server.Items.GargishClothKilt", "Server.Items.GargishClothKiltArmor")] public partial class GargishClothKiltType2 : BaseArmor { [Constructible] diff --git a/Projects/UOContent/Mobiles/Monsters/Reptile/Magic/SkeletalDragon.cs b/Projects/UOContent/Mobiles/Monsters/Reptile/Magic/SkeletalDragon.cs index 774bf438c..e3d5ebb6f 100644 --- a/Projects/UOContent/Mobiles/Monsters/Reptile/Magic/SkeletalDragon.cs +++ b/Projects/UOContent/Mobiles/Monsters/Reptile/Magic/SkeletalDragon.cs @@ -1,5 +1,3 @@ -using System.Collections.Generic; - namespace Server.Mobiles { public class SkeletalDragon : BaseCreature diff --git a/Projects/UOContent/Spells/Chivalry/NobleSacrifice.cs b/Projects/UOContent/Spells/Chivalry/NobleSacrifice.cs index d6e9f4a79..8b67f9b6b 100644 --- a/Projects/UOContent/Spells/Chivalry/NobleSacrifice.cs +++ b/Projects/UOContent/Spells/Chivalry/NobleSacrifice.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using Server.Collections; using Server.Gumps; using Server.Items; diff --git a/version.json b/version.json index 44aefe8f2..00b0a5ef6 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "0.9.3" + "version": "0.9.4" }