warrentyexpired/ModernUO
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(saves): idx v4 with embedded type table

Browse source 
Records reference a per-persistence name table by ushort index instead of a
9-byte tag+hash (33 -> 26 bytes per record), eliminating the per-record
xxHash over Type.FullName on the snapshot thread. The loader resolves each
table name once and indexes records into the constructor array. Legacy
v0-v3 loading is unchanged.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Kamron Batman 2026-07-16 12:03:22 -07:00
parent 4970e22541
commit 98b054ad8d
No known key found for this signature in database
GPG key ID: 7D81DF26D9A5D94A
2 changed files with 152 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

21
Projects/Server.Tests/Tests/Serialization/GenericEntityPersistenceRoundTripTests.cs
View file

@ -100,6 +100,8 @@ public class GenericEntityPersistenceRoundTripTests
};
}
persistence.RegisterType(typeof(RoundTripEntity));
// Freeze: what Persistence.SerializeAll + GenericEntityPersistence.Serialize do,
// against this test's chunk source instead of the world's.
foreach (var worker in workers)
@ -119,22 +121,16 @@ public class GenericEntityPersistenceRoundTripTests
worker.Sleep();
}
// Background write phase: snapshot from the segment logs, then the types db.
var typeSet = new HashSet<Type>();
persistence.WriteSnapshot(dir, typeSet);
var typesDb = new Dictionary<ulong, string>();
foreach (var type in typeSet)
{
typesDb[AssemblyHandler.GetTypeHash(type)] = type.FullName;
}
// Background write phase: snapshot from the segment logs. v4 embeds the type
// table in the idx, so no SerializedTypes.db is produced or needed.
persistence.WriteSnapshot(dir, []);
persistence.PostWorldSave(); // releases the entries snapshot
// Load into a fresh persistence, like a server boot would.
loaded = new RoundTripPersistence(2001);
loaded.DeserializeIndexes(dir, typesDb);
loaded.Deserialize(dir, typesDb);
loaded.DeserializeIndexes(dir, null);
loaded.Deserialize(dir, null);
Assert.True(loaded.SelfPayloadDeserialized);
Assert.Equal(persistence.EntitiesBySerial.Count, loaded.EntitiesBySerial.Count);
@ -146,6 +142,9 @@ public class GenericEntityPersistenceRoundTripTests
Assert.Equal(original.Name, entity.Name);
Assert.Equal(original.Created.Ticks, entity.Created.Ticks);
}
// Loading must hydrate the type table so the next save can write indexes.
Assert.True(loaded.TryGetTypeIndex(typeof(RoundTripEntity), out _));
}
finally
{

189
Projects/Server/Serialization/GenericEntityPersistence.cs
View file

@ -114,8 +114,9 @@ public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPer
using var binFs = new FileStream(
Path.Combine(dir, $"{Name}.bin"), FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024
);
// idx entries are fixed-width: version + count + 33 bytes per record.
var expectedIdxSize = 8 + 33L * EntitiesBySerial.Count;
// v4 records are fixed-width 26 bytes; the header carries the type table
// (name lengths vary — 64 bytes per entry is a staging hint, not a contract).
var expectedIdxSize = 12 + 26L * EntitiesBySerial.Count + 64L * _typeTable.Count;
using var idx = new FileBufferWriter(Path.Combine(dir, $"{Name}.idx"), typeSet, expectedIdxSize);
var binPosition = 0L;
@ -141,7 +142,16 @@ public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPer
binPosition += _selfLength;
}
idx.Write(3); // Version
idx.Write(4); // Version
// The type table is fully known at freeze (AddEntity diverts to the pending
// queues while saving) and is written before the records so the loader can
// resolve constructors before reading them.
idx.Write(_typeTable.Count);
for (var i = 0; i < _typeTable.Count; i++)
{
idx.WriteRaw(_typeTable[i].FullName);
}
var countPosition = idx.Position;
idx.Write(0);
@ -229,7 +239,7 @@ public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPer
continue;
}
idx.Write(entity.GetType());
idx.Write(GetTypeIndex(entity));
idx.Write(entity.Serial);
idx.Write(entity.Created.Ticks);
idx.Write(binPosition);
@ -261,7 +271,7 @@ public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPer
continue;
}
idx.Write(entity.GetType());
idx.Write(GetTypeIndex(entity));
idx.Write(entity.Serial);
idx.Write(entity.Created.Ticks);
idx.Write(binPosition);
@ -281,6 +291,18 @@ public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPer
return binPosition;
}
private ushort GetTypeIndex(T entity)
{
if (!_typeIndexes.TryGetValue(entity.GetType(), out var typeIndex))
{
throw new InvalidOperationException(
$"{entity.GetType()} was serialized but never registered; entities must enter {Name} through AddEntity."
);
}
return typeIndex;
}
public override void Serialize()
{
// Self-payload first so a large one overlaps the entity stream instead of ending it.
@ -401,7 +423,16 @@ public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPer
{
// Legacy didn't have the null flag check
var typeName = dataReader.ReadStringRaw();
types.Add((ulong)i, GetConstructorFor(typeName, AssemblyHandler.FindTypeByName(typeName), ctorArguments));
var type = AssemblyHandler.FindTypeByName(typeName);
var ctor = GetConstructorFor(typeName, type, ctorArguments);
if (ctor != null)
{
// Keep the type table complete so the next (v4) save can index it.
RegisterType(type);
}
types.Add((ulong)i, ctor);
}
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
@ -459,50 +490,121 @@ public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPer
var version = dataReader.ReadInt();
var ctors = version < 2 ? ReadTypes(Path.GetDirectoryName(filePath)) : [];
if (typesDb == null && ctors.Count == 0)
if (version >= 4)
{
return;
DeserializeIndexesV4(dataReader, entities);
}
else
{
var ctors = version < 2 ? ReadTypes(Path.GetDirectoryName(filePath)) : [];
if (typesDb == null && ctors.Count == 0)
{
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
return;
}
var now = DateTime.UtcNow;
var ctorArgs = new object[1];
Type[] ctorArguments = [typeof(Serial)];
var count = dataReader.ReadInt();
for (var i = 0; i < count; ++i)
{
ulong hash;
// Version 2 & 3 with SerializedTypes.db
if (version >= 2)
{
var flag = dataReader.ReadByte();
if (flag != 2)
{
throw new Exception($"Invalid type flag, expected 2 but received {flag}.");
}
hash = dataReader.ReadULong();
}
else
{
hash = (ulong)dataReader.ReadInt(); // Legacy RunUO tdb index
}
if (!ctors.TryGetValue(hash, out var ctor) && typesDb?.TryGetValue(hash, out var typeName) == true)
{
var type = AssemblyHandler.FindTypeByHash(hash);
ctors[hash] = ctor = GetConstructorFor(typeName, type, ctorArguments);
if (ctor != null)
{
// Keep the type table complete so the next (v4) save can index it.
RegisterType(type);
}
}
var serial = (Serial)dataReader.ReadUInt();
var created = version == 0 ? now : new DateTime(dataReader.ReadLong(), DateTimeKind.Utc);
if (version is > 0 and < 3)
{
dataReader.ReadLong(); // LastSerialized
}
var pos = dataReader.ReadLong();
var length = dataReader.ReadInt();
if (ctor == null)
{
continue;
}
ctorArgs[0] = serial;
if (ctor.Invoke(ctorArgs) is T entity)
{
entity.Created = created;
entities.Add(new EntitySpan<T>(entity, pos, length));
EntitiesBySerial[serial] = entity;
}
}
}
var now = DateTime.UtcNow;
var ctorArgs = new object[1];
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
if (EntitiesBySerial.Count > 0)
{
_lastEntitySerial = EntitiesBySerial.Keys.Max();
}
}
private void DeserializeIndexesV4(UnmanagedDataReader dataReader, List<EntitySpan<T>> entities)
{
Type[] ctorArguments = [typeof(Serial)];
var typeCount = dataReader.ReadInt();
var ctors = new ConstructorInfo[typeCount];
for (var i = 0; i < typeCount; i++)
{
var typeName = dataReader.ReadStringRaw();
var type = AssemblyHandler.FindTypeByHash(HashUtility.ComputeHash64(typeName));
var ctor = GetConstructorFor(typeName, type, ctorArguments);
if (ctor != null)
{
// Keep the type table complete so the next save can index it.
RegisterType(type);
}
ctors[i] = ctor;
}
var ctorArgs = new object[1];
var count = dataReader.ReadInt();
for (var i = 0; i < count; ++i)
{
ulong hash;
// Version 2 & 3 with SerializedTypes.db
if (version >= 2)
{
var flag = dataReader.ReadByte();
if (flag != 2)
{
throw new Exception($"Invalid type flag, expected 2 but received {flag}.");
}
hash = dataReader.ReadULong();
}
else
{
hash = (ulong)dataReader.ReadInt(); // Legacy RunUO tdb index
}
if (!ctors.TryGetValue(hash, out var ctor) && typesDb?.TryGetValue(hash, out var typeName) == true)
{
ctors[hash] = ctor = GetConstructorFor(typeName, AssemblyHandler.FindTypeByHash(hash), ctorArguments);
}
var ctor = ctors[dataReader.ReadUShort()];
var serial = (Serial)dataReader.ReadUInt();
var created = version == 0 ? now : new DateTime(dataReader.ReadLong(), DateTimeKind.Utc);
if (version is > 0 and < 3)
{
dataReader.ReadLong(); // LastSerialized
}
var created = new DateTime(dataReader.ReadLong(), DateTimeKind.Utc);
var pos = dataReader.ReadLong();
var length = dataReader.ReadInt();
@ -520,13 +622,6 @@ public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPer
EntitiesBySerial[serial] = entity;
}
}
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
if (EntitiesBySerial.Count > 0)
{
_lastEntitySerial = EntitiesBySerial.Keys.Max();
}
}
public override void Deserialize(string savePath, Dictionary<ulong, string> typesDb)