fix: Adds generic entity persistence support and BOBEntry as entities (#1527)

### Summary
Adds a generic entity persistence. This can be used to create new entity types that have a `Serial`.

Here is an example:

```cs
public class BOBEntries : GenericEntitySerialization<IBOBEntry>
{
    public static void Configure()
    {
        Configure("BOBEntries");
    }
}
```

The annotation tells the system what folder to serialize the entries to. The class/interface (`IBOBEntry`) is the root type that implements `ISerializable`.

Projects/Server/Guild.cs

@@ -50,9 +50,6 @@ public abstract class BaseGuild : ISerializable
     [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
     public DateTime Created { get; set; } = Core.Now;
 
-    [CommandProperty(AccessLevel.GameMaster)]
-    DateTime ISerializable.LastSerialized { get; set; } = Core.Now;
-
     long ISerializable.SavePosition { get; set; } = -1;
 
     BufferWriter ISerializable.SaveBuffer { get; set; }

Projects/Server/IEntity.cs

@@ -45,8 +45,6 @@ public class Entity : IEntity
 
     DateTime ISerializable.Created { get; set; } = Core.Now;
 
-    DateTime ISerializable.LastSerialized { get; set; } = DateTime.MaxValue;
-
     long ISerializable.SavePosition { get; set; } = -1;
 
     BufferWriter ISerializable.SaveBuffer { get; set; }

Projects/Server/Items/Item.cs

@@ -760,9 +760,6 @@ public class Item : IHued, IComparable<Item>, ISpawnable, IObjectPropertyListEnt
     [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
     public DateTime Created { get; set; } = Core.Now;
 
-    [CommandProperty(AccessLevel.GameMaster)]
-    DateTime ISerializable.LastSerialized { get; set; } = Core.Now;
-
     long ISerializable.SavePosition { get; set; } = -1;
 
     BufferWriter ISerializable.SaveBuffer { get; set; }

Projects/Server/Mobiles/Mobile.cs

@@ -2258,9 +2258,6 @@ public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPro
     [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
     public DateTime Created { get; set; } = Core.Now;
 
-    [CommandProperty(AccessLevel.GameMaster)]
-    DateTime ISerializable.LastSerialized { get; set; } = Core.Now;
-
     long ISerializable.SavePosition { get; set; } = -1;
 
     BufferWriter ISerializable.SaveBuffer { get; set; }

Projects/Server/Serialization/GenericEntitySerialization.cs

@@ -0,0 +1,209 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using Server.Logging;
+
+namespace Server;
+
+public class GenericEntitySerialization<T> where T : class, ISerializable
+{
+    private static readonly ILogger logger = LogFactory.GetLogger(typeof(GenericEntitySerialization<T>));
+
+    private static string _systemName;
+    private static Serial _lastEntitySerial;
+    private static readonly Dictionary<Serial, T> _pendingAdd = new();
+    private static readonly Dictionary<Serial, T> _pendingDelete = new();
+    private static Dictionary<Serial, T> _entitiesBySerial = new();
+
+    public static void Configure(string systemName)
+    {
+        _systemName = systemName;
+        typeof(T).RegisterFindEntity(Find);
+        Persistence.Register(_systemName, Serialize, WriteSnapshot, Deserialize);
+    }
+
+    internal static void Serialize()
+    {
+        EntityPersistence.SaveEntities(
+            _entitiesBySerial.Values,
+            entity => entity.Serialize(World.SerializedTypes)
+        );
+    }
+
+    internal static void WriteSnapshot(string basePath)
+    {
+        IIndexInfo<Serial> indexInfo = new EntityTypeIndex(_systemName);
+        EntityPersistence.WriteEntities(indexInfo, _entitiesBySerial, basePath,World.SerializedTypes, out _);
+    }
+
+    internal static void Deserialize(string path, Dictionary<ulong, string> typesDb)
+    {
+        IIndexInfo<Serial> indexInfo = new EntityTypeIndex(_systemName);
+
+        _entitiesBySerial = EntityPersistence.LoadIndex(path, indexInfo, typesDb, out List<EntitySpan<T>> entities);
+
+        if (_entitiesBySerial.Count > 0)
+        {
+            _lastEntitySerial = _entitiesBySerial.Keys.Max();
+        }
+
+        EntityPersistence.LoadData(path, indexInfo, typesDb, entities);
+    }
+
+    public static Serial NewEntity
+    {
+        get
+        {
+#if THREADGUARD
+            if (Thread.CurrentThread != Core.Thread)
+            {
+                logger.Error(
+                    "Attempted to get a new entity serial from the wrong thread!\n{StackTrace}",
+                    new StackTrace()
+                );
+            }
+#endif
+            var last = _lastEntitySerial;
+
+            for (uint i = 0; i < uint.MaxValue; i++)
+            {
+                last++;
+
+                if (FindEntity<T>(last) == null)
+                {
+                    return _lastEntitySerial = last;
+                }
+            }
+
+            OutOfMemory("No serials left to allocate for BOBEntries");
+            return Serial.MinusOne;
+        }
+    }
+
+    public static void AddEntity(T entity)
+    {
+        var worldState = World.WorldState;
+        switch (worldState)
+        {
+            default: // Not Running
+                {
+                    throw new Exception($"Added {entity.GetType().Name} before world load.");
+                }
+            case WorldState.Saving:
+            case WorldState.Loading:
+            case WorldState.WritingSave:
+                {
+                    if (_pendingDelete.Remove(entity.Serial))
+                    {
+                        logger.Warning("Deleted then added {Entity} during {WorldState} state.", entity.GetType().Name, worldState.ToString());
+                    }
+
+                    _pendingAdd[entity.Serial] = entity;
+                    break;
+                }
+            case WorldState.Running:
+                {
+                    ref var entityEntry = ref CollectionsMarshal.GetValueRefOrAddDefault(_entitiesBySerial, entity.Serial, out bool exists);
+                    if (exists)
+                    {
+                        if (entityEntry == entity)
+                        {
+                            logger.Error(
+                                $"Attempted to add '{{Entity}}' ({{Serial}}) to World.Items but it already exists in the collection.{Environment.NewLine}{{StackTrace}}",
+                                entity.GetType().FullName,
+                                entity.Serial,
+                                new StackTrace()
+                            );
+                        }
+                        else
+                        {
+                            logger.Error(
+                                $"Attempted to add '{{Entity}}' ({{Serial}}) to World.Items but found '{{ExistingEntity}}' ({{ExistingSerial}}).{Environment.NewLine}{{StackTrace}}",
+                                entity.GetType().FullName,
+                                entity.Serial,
+                                entityEntry.GetType().FullName,
+                                entityEntry.Serial,
+                                new StackTrace()
+                            );
+                        }
+                    }
+                    else
+                    {
+                        entityEntry = entity;
+                    }
+                    break;
+                }
+        }
+    }
+
+    public static void RemoveEntity(T entity)
+    {
+        var worldState = World.WorldState;
+        switch (worldState)
+        {
+            default: // Not Running
+                {
+                    throw new Exception($"Removed {entity.GetType().Name} before world load.");
+                }
+            case WorldState.Saving:
+            case WorldState.Loading:
+            case WorldState.WritingSave:
+                {
+                    _pendingAdd.Remove(entity.Serial);
+                    _pendingDelete[entity.Serial] = entity;
+                    break;
+                }
+            case WorldState.Running:
+                {
+                    _entitiesBySerial.Remove(entity.Serial);
+                    break;
+                }
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static T Find(Serial serial) => FindEntity<T>(serial, false, false);
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static T Find(Serial serial, bool returnDeleted) => FindEntity<T>(serial, returnDeleted, false);
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static T Find(Serial serial, bool returnDeleted, bool returnPending) => FindEntity<T>(serial, returnDeleted, returnPending);
+
+    public static R FindEntity<R>(Serial serial) where R : class, T => FindEntity<R>(serial, false, false);
+
+    public static R FindEntity<R>(Serial serial, bool returnDeleted, bool returnPending) where R : class, T
+    {
+        switch (World.WorldState)
+        {
+            default: return null;
+            case WorldState.Loading:
+            case WorldState.Saving:
+            case WorldState.WritingSave:
+                {
+                    if (returnDeleted && returnPending && _pendingDelete.TryGetValue(serial, out var entity))
+                    {
+                        return entity as R;
+                    }
+
+                    if (returnPending && _pendingAdd.TryGetValue(serial, out entity) ||
+                        _entitiesBySerial.TryGetValue(serial, out entity))
+                    {
+                        return entity as R;
+                    }
+
+                    return null;
+                }
+            case WorldState.Running:
+                {
+                    return _entitiesBySerial.TryGetValue(serial, out var entity) ? entity as R : null;
+                }
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static void OutOfMemory(string message) => throw new OutOfMemoryException(message);
+}

Projects/Server/Serialization/ISerializable.cs

@@ -24,8 +24,6 @@ public interface ISerializable
     // Should be serialized/deserialized with the index so it can be referenced by IGenericReader
     DateTime Created { get; set; }
 
-    // Should be serialized/deserialized with the index so it can be referenced by IGenericReader
-    DateTime LastSerialized { get; protected internal set; }
     long SavePosition { get; protected internal set; }
     BufferWriter SaveBuffer { get; protected internal set; }
 
@@ -61,7 +59,6 @@ public interface ISerializable
             return;
         }
 
-        LastSerialized = Core.Now;
         SaveBuffer.Seek(0, SeekOrigin.Begin);
         Serialize(SaveBuffer);
 

Projects/Server/Serialization/SerializationExtensions.cs

@@ -21,6 +21,14 @@ namespace Server;
 
 public static class SerializationExtensions
 {
+    private static readonly Dictionary<Type, Func<Serial, bool, bool, ISerializable>> _directFinderTable = new();
+    private static readonly Dictionary<Type, Func<Serial, bool, bool, ISerializable>> _searchTable = new();
+
+    public static void RegisterFindEntity(this Type type, Func<Serial, bool, bool, ISerializable> func)
+    {
+        _searchTable[type] = func;
+    }
+
     public static T ReadEntity<T>(this IGenericReader reader) where T : class, ISerializable
     {
         Serial serial = reader.ReadSerial();
@@ -31,21 +39,55 @@ public static class SerializationExtensions
         // Add to this list when creating new serializable types
         if (typeof(BaseGuild).IsAssignableFrom(typeT))
         {
-            entity = World.FindGuild(serial) as T;
             // If we check for `entity.Deleted` here during deserialization then all guilds are deleted because
             // Deleted -> Disbanded -> No leader, which is the case before deserialization.
-            // TODO: Use a deleted flag instead, and actively check for dibanded guilds properly.
+            // TODO: Use a deleted flag instead, and actively check for disbanded guilds properly.
+            return World.FindGuild(serial) as T;
         }
-        else
+
+        if (typeof(IEntity).IsAssignableFrom(typeT))
         {
-            entity = World.FindEntity<IEntity>(serial) as T;
-            if (entity?.Deleted == false)
+            return World.FindEntity<IEntity>(serial, returnPending: false) as T;
+        }
+
+        if (_directFinderTable.TryGetValue(typeT, out var finder))
+        {
+            return finder(serial, false, false) as T;
+        }
+
+        Type type = null;
+        foreach (var baseType in _searchTable.Keys)
+        {
+            if (baseType.IsAssignableFrom(typeT))
             {
-                return entity;
+                type = baseType;
+                break;
             }
         }
 
-        return entity?.Created <= reader.LastSerialized ? entity : null;
+        if (type == null)
+        {
+            type = typeT;
+            while (true)
+            {
+                var baseType = type?.BaseType;
+
+                // Find the parent class with ISerializable registered. To do this we break on it's parent class (or object)
+                // that doesn't have ISerializable implemented.
+                if (baseType?.GetInterface("ISerializable") == null && type?.GetInterface("ISerializable") != null)
+                {
+                    break;
+                }
+
+                type = baseType;
+            }
+
+            throw new Exception($"No FindEntity registered for '{type.FullName}'.");
+        }
+
+        finder = _searchTable[type];
+        _directFinderTable[type] = finder;
+        return finder(serial, false, false) as T;
     }
 
     public static List<T> ReadEntityList<T>(

Projects/Server/World/EntityPersistence.cs

@@ -48,7 +48,7 @@ public static class EntityPersistence
         using var idx = new BinaryFileWriter(idxPath, false, types);
         using var bin = new BinaryFileWriter(binPath, true, types);
 
-        idx.Write(2); // Version
+        idx.Write(3); // Version
         idx.Write(entities.Count);
         foreach (var e in entities.Values)
         {
@@ -58,7 +58,6 @@ public static class EntityPersistence
             idx.Write(t);
             idx.Write(e.Serial);
             idx.Write(e.Created.Ticks);
-            idx.Write(e.LastSerialized.Ticks);
             idx.Write(start);
 
             e.SerializeTo(bin);
@@ -152,7 +151,10 @@ public static class EntityPersistence
 
             var serial = idxReader.ReadUInt32();
             var created = version == 0 ? now : new DateTime(idxReader.ReadInt64(), DateTimeKind.Utc);
-            var lastSerialized = version == 0 ? DateTime.MinValue : new DateTime(idxReader.ReadInt64(), DateTimeKind.Utc);
+            if (version is > 0 and < 3)
+            {
+                idxReader.ReadInt64(); // LastSerialized
+            }
             var pos = idxReader.ReadInt64();
             var length = idxReader.ReadInt32();
 
@@ -168,7 +170,6 @@ public static class EntityPersistence
             if (ctor.Invoke(ctorArgs) is T entity)
             {
                 entity.Created = created;
-                entity.LastSerialized = lastSerialized;
                 entities.Add(new EntitySpan<T>(entity, pos, length));
                 map[indexer] = entity;
             }
@@ -223,7 +224,7 @@ public static class EntityPersistence
             var buffer = GC.AllocateUninitializedArray<byte>(entry.Length);
             if (br == null)
             {
-                br = new BufferReader(buffer, t.LastSerialized, serializedTypes);
+                br = new BufferReader(buffer, serializedTypes);
             }
             else
             {

Projects/Server/World/World.cs

@@ -20,6 +20,7 @@ using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Threading;
 using Server.Guilds;
 using Server.Logging;
@@ -592,9 +593,10 @@ public static class World
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static IEntity FindEntity(Serial serial, bool returnDeleted = false) => FindEntity<IEntity>(serial, returnDeleted);
+    public static IEntity FindEntity(Serial serial, bool returnDeleted = false, bool returnPending = true) =>
+        FindEntity<IEntity>(serial, returnDeleted, returnPending);
 
-    public static T FindEntity<T>(Serial serial, bool returnDeleted = false) where T : class, IEntity
+    public static T FindEntity<T>(Serial serial, bool returnDeleted = false, bool returnPending = true) where T : class, IEntity
     {
         switch (WorldState)
         {
@@ -603,43 +605,41 @@ public static class World
             case WorldState.Saving:
             case WorldState.WritingSave:
                 {
-                    if (returnDeleted && _pendingDelete.TryGetValue(serial, out var entity))
+                    if (returnDeleted && returnPending && _pendingDelete.TryGetValue(serial, out var entity))
                     {
                         return entity as T;
                     }
 
-                    if (!_pendingAdd.TryGetValue(serial, out entity))
+                    if (!returnPending || !_pendingAdd.TryGetValue(serial, out entity))
                     {
                         if (serial.IsItem)
                         {
                             if (Items.TryGetValue(serial, out var item))
                             {
-                                entity = item;
+                                return item as T;
                             }
                         }
                         else // if (serial.IsMobile)
                         {
                             if (Mobiles.TryGetValue(serial, out var mob))
                             {
-                                entity = mob;
+                                return mob as T;
                             }
                         }
                     }
 
-                    return entity?.Deleted == false || returnDeleted ? entity as T : null;
+                    return null;
                 }
             case WorldState.Running:
                 {
                     if (serial.IsItem)
                     {
-                        Items.TryGetValue(serial, out var item);
-                        return item as T;
+                        return Items.TryGetValue(serial, out var item) ? item as T : null;
                     }
 
                     if (serial.IsMobile)
                     {
-                        Mobiles.TryGetValue(serial, out var mob);
-                        return mob as T;
+                        return Mobiles.TryGetValue(serial, out var mob) ? mob as T : null;
                     }
 
                     return default;
@@ -684,11 +684,10 @@ public static class World
                 {
                     if (entity.Serial.IsItem)
                     {
-                        if (!Items.TryAdd(entity.Serial, entity as Item))
+                        ref var item = ref CollectionsMarshal.GetValueRefOrAddDefault(Items, entity.Serial, out bool exists);
+                        if (exists)
                         {
-                            var existing = Items[entity.Serial];
-
-                            if (existing == entity)
+                            if (item == entity)
                             {
                                 logger.Error(
                                     $"Attempted to add '{{Entity}}' ({{Serial}}) to World.Items but it already exists in the collection.{Environment.NewLine}{{StackTrace}}",
@@ -703,21 +702,24 @@ public static class World
                                     $"Attempted to add '{{Entity}}' ({{Serial}}) to World.Items but found '{{ExistingEntity}}' ({{ExistingSerial}}).{Environment.NewLine}{{StackTrace}}",
                                     entity.GetType().FullName,
                                     entity.Serial,
-                                    existing.GetType().FullName,
-                                    existing.Serial,
+                                    item.GetType().FullName,
+                                    item.Serial,
                                     new StackTrace()
                                 );
                             }
                         }
+                        else
+                        {
+                            item = entity as Item;
+                        }
                     }
 
                     if (entity.Serial.IsMobile)
                     {
-                        if (!Mobiles.TryAdd(entity.Serial, entity as Mobile))
+                        ref var mob = ref CollectionsMarshal.GetValueRefOrAddDefault(Mobiles, entity.Serial, out bool exists);
+                        if (exists)
                         {
-                            var existing = Mobiles[entity.Serial];
-
-                            if (existing == entity)
+                            if (mob == entity)
                             {
                                 logger.Error(
                                     $"Attempted to add '{{Entity}}' ({{Serial}}) to World.Mobiles but it already exists in the collection.{Environment.NewLine}{{StackTrace}}",
@@ -732,30 +734,33 @@ public static class World
                                     $"Attempted to add '{{Entity}}' ({{Serial}}) to World.Mobiles but found '{{ExistingEntity}}' ({{ExistingSerial}}).{Environment.NewLine}{{StackTrace}}",
                                     entity.GetType().FullName,
                                     entity.Serial,
-                                    existing.GetType().FullName,
-                                    existing.Serial,
+                                    mob.GetType().FullName,
+                                    mob.Serial,
                                     new StackTrace()
                                 );
                             }
                         }
+                        else
+                        {
+                            mob = entity as Mobile;
+                        }
                     }
                     break;
                 }
         }
     }
 
-    public static void AddGuild(BaseGuild guild)
+    public static void AddGuild(BaseGuild entity)
     {
-        if (!Guilds.TryAdd(guild.Serial, guild))
+        ref var guild = ref CollectionsMarshal.GetValueRefOrAddDefault(Guilds, entity.Serial, out bool exists);
+        if (exists)
         {
-            var existing = Guilds[guild.Serial];
-
-            if (existing == guild)
+            if (guild == entity)
             {
                 logger.Error(
                     $"Attempted to add '{{Entity}}' ({{Serial}}) to World.Guilds but it already exists in the collection.{Environment.NewLine}{{StackTrace}}",
-                    guild.GetType().FullName,
-                    guild.Serial,
+                    entity.GetType().FullName,
+                    entity.Serial,
                     new StackTrace()
                 );
             }
@@ -763,14 +768,18 @@ public static class World
             {
                 logger.Error(
                     $"Attempted to add '{{Entity}}' ({{Serial}}) to World.Guilds but found '{{ExistingEntity}}' ({{ExistingSerial}}).{Environment.NewLine}{{StackTrace}}",
+                    entity.GetType().FullName,
+                    entity.Serial,
                     guild.GetType().FullName,
                     guild.Serial,
-                    existing.GetType().FullName,
-                    existing.Serial,
                     new StackTrace()
                 );
             }
         }
+        else
+        {
+            guild = entity;
+        }
     }
 
     public static void RemoveEntity<T>(T entity) where T : class, IEntity