From 98b054ad8d43097df87620e093b2deaeadfee01d Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:03:22 -0700 Subject: [PATCH] feat(saves): idx v4 with embedded type table Records reference a per-persistence name table by ushort index instead of a 9-byte tag+hash (33 -> 26 bytes per record), eliminating the per-record xxHash over Type.FullName on the snapshot thread. The loader resolves each table name once and indexes records into the constructor array. Legacy v0-v3 loading is unchanged. Co-Authored-By: Claude Fable 5 --- .../GenericEntityPersistenceRoundTripTests.cs | 21 +- .../Serialization/GenericEntityPersistence.cs | 189 +++++++++++++----- 2 files changed, 152 insertions(+), 58 deletions(-) diff --git a/Projects/Server.Tests/Tests/Serialization/GenericEntityPersistenceRoundTripTests.cs b/Projects/Server.Tests/Tests/Serialization/GenericEntityPersistenceRoundTripTests.cs index eeedd9241..9faf986f9 100644 --- a/Projects/Server.Tests/Tests/Serialization/GenericEntityPersistenceRoundTripTests.cs +++ b/Projects/Server.Tests/Tests/Serialization/GenericEntityPersistenceRoundTripTests.cs @@ -100,6 +100,8 @@ public class GenericEntityPersistenceRoundTripTests }; } + persistence.RegisterType(typeof(RoundTripEntity)); + // Freeze: what Persistence.SerializeAll + GenericEntityPersistence.Serialize do, // against this test's chunk source instead of the world's. foreach (var worker in workers) @@ -119,22 +121,16 @@ public class GenericEntityPersistenceRoundTripTests worker.Sleep(); } - // Background write phase: snapshot from the segment logs, then the types db. - var typeSet = new HashSet(); - persistence.WriteSnapshot(dir, typeSet); - - var typesDb = new Dictionary(); - foreach (var type in typeSet) - { - typesDb[AssemblyHandler.GetTypeHash(type)] = type.FullName; - } + // Background write phase: snapshot from the segment logs. v4 embeds the type + // table in the idx, so no SerializedTypes.db is produced or needed. + persistence.WriteSnapshot(dir, []); persistence.PostWorldSave(); // releases the entries snapshot // Load into a fresh persistence, like a server boot would. loaded = new RoundTripPersistence(2001); - loaded.DeserializeIndexes(dir, typesDb); - loaded.Deserialize(dir, typesDb); + loaded.DeserializeIndexes(dir, null); + loaded.Deserialize(dir, null); Assert.True(loaded.SelfPayloadDeserialized); Assert.Equal(persistence.EntitiesBySerial.Count, loaded.EntitiesBySerial.Count); @@ -146,6 +142,9 @@ public class GenericEntityPersistenceRoundTripTests Assert.Equal(original.Name, entity.Name); Assert.Equal(original.Created.Ticks, entity.Created.Ticks); } + + // Loading must hydrate the type table so the next save can write indexes. + Assert.True(loaded.TryGetTypeIndex(typeof(RoundTripEntity), out _)); } finally { diff --git a/Projects/Server/Serialization/GenericEntityPersistence.cs b/Projects/Server/Serialization/GenericEntityPersistence.cs index 8044fe97b..5ee60c5e4 100644 --- a/Projects/Server/Serialization/GenericEntityPersistence.cs +++ b/Projects/Server/Serialization/GenericEntityPersistence.cs @@ -114,8 +114,9 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer using var binFs = new FileStream( Path.Combine(dir, $"{Name}.bin"), FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 ); - // idx entries are fixed-width: version + count + 33 bytes per record. - var expectedIdxSize = 8 + 33L * EntitiesBySerial.Count; + // v4 records are fixed-width 26 bytes; the header carries the type table + // (name lengths vary — 64 bytes per entry is a staging hint, not a contract). + var expectedIdxSize = 12 + 26L * EntitiesBySerial.Count + 64L * _typeTable.Count; using var idx = new FileBufferWriter(Path.Combine(dir, $"{Name}.idx"), typeSet, expectedIdxSize); var binPosition = 0L; @@ -141,7 +142,16 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer binPosition += _selfLength; } - idx.Write(3); // Version + idx.Write(4); // Version + + // The type table is fully known at freeze (AddEntity diverts to the pending + // queues while saving) and is written before the records so the loader can + // resolve constructors before reading them. + idx.Write(_typeTable.Count); + for (var i = 0; i < _typeTable.Count; i++) + { + idx.WriteRaw(_typeTable[i].FullName); + } var countPosition = idx.Position; idx.Write(0); @@ -229,7 +239,7 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer continue; } - idx.Write(entity.GetType()); + idx.Write(GetTypeIndex(entity)); idx.Write(entity.Serial); idx.Write(entity.Created.Ticks); idx.Write(binPosition); @@ -261,7 +271,7 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer continue; } - idx.Write(entity.GetType()); + idx.Write(GetTypeIndex(entity)); idx.Write(entity.Serial); idx.Write(entity.Created.Ticks); idx.Write(binPosition); @@ -281,6 +291,18 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer return binPosition; } + private ushort GetTypeIndex(T entity) + { + if (!_typeIndexes.TryGetValue(entity.GetType(), out var typeIndex)) + { + throw new InvalidOperationException( + $"{entity.GetType()} was serialized but never registered; entities must enter {Name} through AddEntity." + ); + } + + return typeIndex; + } + public override void Serialize() { // Self-payload first so a large one overlaps the entity stream instead of ending it. @@ -401,7 +423,16 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer { // Legacy didn't have the null flag check var typeName = dataReader.ReadStringRaw(); - types.Add((ulong)i, GetConstructorFor(typeName, AssemblyHandler.FindTypeByName(typeName), ctorArguments)); + var type = AssemblyHandler.FindTypeByName(typeName); + var ctor = GetConstructorFor(typeName, type, ctorArguments); + + if (ctor != null) + { + // Keep the type table complete so the next (v4) save can index it. + RegisterType(type); + } + + types.Add((ulong)i, ctor); } accessor.SafeMemoryMappedViewHandle.ReleasePointer(); @@ -459,50 +490,121 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer var version = dataReader.ReadInt(); - var ctors = version < 2 ? ReadTypes(Path.GetDirectoryName(filePath)) : []; - - if (typesDb == null && ctors.Count == 0) + if (version >= 4) { - return; + DeserializeIndexesV4(dataReader, entities); + } + else + { + var ctors = version < 2 ? ReadTypes(Path.GetDirectoryName(filePath)) : []; + + if (typesDb == null && ctors.Count == 0) + { + accessor.SafeMemoryMappedViewHandle.ReleasePointer(); + return; + } + + var now = DateTime.UtcNow; + var ctorArgs = new object[1]; + Type[] ctorArguments = [typeof(Serial)]; + + var count = dataReader.ReadInt(); + + for (var i = 0; i < count; ++i) + { + ulong hash; + // Version 2 & 3 with SerializedTypes.db + if (version >= 2) + { + var flag = dataReader.ReadByte(); + if (flag != 2) + { + throw new Exception($"Invalid type flag, expected 2 but received {flag}."); + } + + hash = dataReader.ReadULong(); + } + else + { + hash = (ulong)dataReader.ReadInt(); // Legacy RunUO tdb index + } + + if (!ctors.TryGetValue(hash, out var ctor) && typesDb?.TryGetValue(hash, out var typeName) == true) + { + var type = AssemblyHandler.FindTypeByHash(hash); + ctors[hash] = ctor = GetConstructorFor(typeName, type, ctorArguments); + + if (ctor != null) + { + // Keep the type table complete so the next (v4) save can index it. + RegisterType(type); + } + } + + var serial = (Serial)dataReader.ReadUInt(); + var created = version == 0 ? now : new DateTime(dataReader.ReadLong(), DateTimeKind.Utc); + if (version is > 0 and < 3) + { + dataReader.ReadLong(); // LastSerialized + } + + var pos = dataReader.ReadLong(); + var length = dataReader.ReadInt(); + + if (ctor == null) + { + continue; + } + + ctorArgs[0] = serial; + + if (ctor.Invoke(ctorArgs) is T entity) + { + entity.Created = created; + entities.Add(new EntitySpan(entity, pos, length)); + EntitiesBySerial[serial] = entity; + } + } } - var now = DateTime.UtcNow; - var ctorArgs = new object[1]; + accessor.SafeMemoryMappedViewHandle.ReleasePointer(); + + if (EntitiesBySerial.Count > 0) + { + _lastEntitySerial = EntitiesBySerial.Keys.Max(); + } + } + + private void DeserializeIndexesV4(UnmanagedDataReader dataReader, List> entities) + { Type[] ctorArguments = [typeof(Serial)]; + var typeCount = dataReader.ReadInt(); + var ctors = new ConstructorInfo[typeCount]; + + for (var i = 0; i < typeCount; i++) + { + var typeName = dataReader.ReadStringRaw(); + var type = AssemblyHandler.FindTypeByHash(HashUtility.ComputeHash64(typeName)); + var ctor = GetConstructorFor(typeName, type, ctorArguments); + + if (ctor != null) + { + // Keep the type table complete so the next save can index it. + RegisterType(type); + } + + ctors[i] = ctor; + } + + var ctorArgs = new object[1]; var count = dataReader.ReadInt(); for (var i = 0; i < count; ++i) { - ulong hash; - // Version 2 & 3 with SerializedTypes.db - if (version >= 2) - { - var flag = dataReader.ReadByte(); - if (flag != 2) - { - throw new Exception($"Invalid type flag, expected 2 but received {flag}."); - } - - hash = dataReader.ReadULong(); - } - else - { - hash = (ulong)dataReader.ReadInt(); // Legacy RunUO tdb index - } - - if (!ctors.TryGetValue(hash, out var ctor) && typesDb?.TryGetValue(hash, out var typeName) == true) - { - ctors[hash] = ctor = GetConstructorFor(typeName, AssemblyHandler.FindTypeByHash(hash), ctorArguments); - } - + var ctor = ctors[dataReader.ReadUShort()]; var serial = (Serial)dataReader.ReadUInt(); - var created = version == 0 ? now : new DateTime(dataReader.ReadLong(), DateTimeKind.Utc); - if (version is > 0 and < 3) - { - dataReader.ReadLong(); // LastSerialized - } - + var created = new DateTime(dataReader.ReadLong(), DateTimeKind.Utc); var pos = dataReader.ReadLong(); var length = dataReader.ReadInt(); @@ -520,13 +622,6 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer EntitiesBySerial[serial] = entity; } } - - accessor.SafeMemoryMappedViewHandle.ReleasePointer(); - - if (EntitiesBySerial.Count > 0) - { - _lastEntitySerial = EntitiesBySerial.Keys.Max(); - } } public override void Deserialize(string savePath, Dictionary typesDb)