diff --git a/Projects/Server.Tests/Tests/Serialization/SerializationChunkSourceTests.cs b/Projects/Server.Tests/Tests/Serialization/SerializationChunkSourceTests.cs new file mode 100644 index 000000000..6ddfae427 --- /dev/null +++ b/Projects/Server.Tests/Tests/Serialization/SerializationChunkSourceTests.cs @@ -0,0 +1,222 @@ +using System.Collections.Generic; +using Xunit; + +namespace Server.Tests; + +public class SerializationChunkSourceTests +{ + private class TestEntity : IGenericSerializable + { + public byte SerializedThread { get; set; } + public int SerializedPosition { get; set; } + public int SerializedLength { get; set; } + + public int PayloadSize { get; init; } = 16; + + public void Serialize(IGenericWriter writer) + { + for (var i = 0; i < PayloadSize; i++) + { + writer.Write((byte)(i & 0xFF)); + } + } + } + + private static List Drain(SerializationChunkSource source) + { + var drained = new List(); + while (source.TryTake(out var chunk)) + { + if (chunk.Single != null) + { + drained.Add((TestEntity)chunk.Single); + } + else + { + for (var i = 0; i < chunk.Count; i++) + { + drained.Add((TestEntity)chunk.Buffer[i]); + } + + source.Return(chunk.Buffer, chunk.Count); + } + } + + return drained; + } + + [Fact] + public void PartialChunkIsNotVisibleUntilFlush() + { + var source = new SerializationChunkSource(); + var entities = new List(); + + for (var i = 0; i < 100; i++) + { + var e = new TestEntity(); + entities.Add(e); + source.Push(e); + } + + Assert.False(source.TryTake(out _)); + + source.Flush(); + + var drained = Drain(source); + Assert.Equal(entities, drained); + + // Flush again should publish nothing + source.Flush(); + Assert.False(source.TryTake(out _)); + } + + [Fact] + public void FullChunkPublishesWithoutFlush() + { + var source = new SerializationChunkSource(); + + for (var i = 0; i < 4096; i++) + { + source.Push(new TestEntity()); + } + + Assert.True(source.TryTake(out var chunk)); + Assert.Null(chunk.Single); + Assert.Equal(4096, chunk.Count); + source.Return(chunk.Buffer, chunk.Count); + + // Nothing partial left behind + source.Flush(); + Assert.False(source.TryTake(out _)); + } + + [Fact] + public void PushSingleDoesNotDisturbPartialChunk() + { + var source = new SerializationChunkSource(); + + var small1 = new TestEntity(); + var heavy = new TestEntity { SerializedLength = 2 * 1024 * 1024 }; + var small2 = new TestEntity(); + + source.Push(small1); + source.PushSingle(heavy); // published immediately as a single, ahead of the partial chunk + source.Push(small2); + + Assert.True(source.TryTake(out var chunk)); + Assert.Same(heavy, chunk.Single); + Assert.Equal(1, chunk.Count); + + Assert.False(source.TryTake(out _)); + source.Flush(); + + var drained = Drain(source); + Assert.Equal([small1, small2], drained); + } + + [Fact] + public void ReturnedBuffersAreClearedAndReused() + { + var source = new SerializationChunkSource(); + + for (var i = 0; i < 4096; i++) + { + source.Push(new TestEntity()); + } + + Assert.True(source.TryTake(out var chunk)); + var buffer = chunk.Buffer; + source.Return(buffer, chunk.Count); + + Assert.All(buffer, Assert.Null); + + // Next fill rents the pooled buffer instead of allocating + source.Push(new TestEntity()); + source.Flush(); + Assert.True(source.TryTake(out var reused)); + Assert.Same(buffer, reused.Buffer); + } + + [Fact] + public void WorkersDrainAllEntitiesAndStampPositions() + { + var source = new SerializationChunkSource(); + var workers = new SerializationThreadWorker[2]; + for (var i = 0; i < workers.Length; i++) + { + workers[i] = new SerializationThreadWorker(i, source); + workers[i].AllocateHeap(); + } + + try + { + var entities = new List(); + for (var i = 0; i < 10_000; i++) + { + entities.Add(new TestEntity { PayloadSize = 16 + i % 64 }); + } + + // A heavy entity mid-stream gets a dedicated chunk + var heavy = new TestEntity { PayloadSize = 512 * 1024, SerializedLength = 4 * 1024 * 1024 }; + entities.Insert(5000, heavy); + + foreach (var worker in workers) + { + worker.Wake(); + } + + // Mirrors GenericEntityPersistence.Serialize: heavy check at the call site + foreach (var e in entities) + { + if (e.SerializedLength > SerializationChunkSource.HeavyEntityThreshold) + { + source.PushSingle(e); + } + else + { + source.Push(e); + } + } + + // Mirrors World.PauseSerializationThreads + source.Flush(); + foreach (var worker in workers) + { + worker.Sleep(); + } + + long totalEntities = 0; + long totalBytes = 0; + foreach (var worker in workers) + { + totalEntities += worker.EntitiesSerialized; + totalBytes += worker.BytesSerialized; + } + + Assert.Equal(entities.Count, totalEntities); + + long expectedBytes = 0; + foreach (var e in entities) + { + expectedBytes += e.PayloadSize; + + // Every entity serialized exactly once with a consistent span on its worker's heap + Assert.Equal(e.PayloadSize, e.SerializedLength); + Assert.InRange(e.SerializedThread, (byte)0, (byte)(workers.Length - 1)); + + var heap = workers[e.SerializedThread].GetHeap(e.SerializedPosition, e.SerializedLength); + Assert.Equal(0, heap[0]); + Assert.Equal((e.SerializedLength - 1) & 0xFF, heap[^1]); + } + + Assert.Equal(expectedBytes, totalBytes); + } + finally + { + foreach (var worker in workers) + { + worker.Exit(); + } + } + } +} diff --git a/Projects/Server/Serialization/GenericEntityPersistence.cs b/Projects/Server/Serialization/GenericEntityPersistence.cs index 501e84aeb..c28fd6a0c 100644 --- a/Projects/Server/Serialization/GenericEntityPersistence.cs +++ b/Projects/Server/Serialization/GenericEntityPersistence.cs @@ -70,7 +70,11 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer var threads = World._threadWorkers; - using var binFs = new FileStream(Path.Combine(dir, $"{Name}.bin"), FileMode.Create, FileAccess.Write, FileShare.None); + // 1MB buffer: entity spans are written individually, so the default 4KB buffer + // costs a syscall every few entities on the snapshot thread. + using var binFs = new FileStream( + Path.Combine(dir, $"{Name}.bin"), FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 + ); using var idxFs = new FileStream(Path.Combine(dir, $"{Name}.idx"), FileMode.Create); using var idx = new MemoryMapFileWriter(idxFs, 1024 * 1024, typeSet); // 1MB @@ -148,12 +152,23 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer public override void Serialize() { + // Self-payload first so a large one overlaps the entity stream instead of ending it. + World.PushSingleToCache(this); + foreach (var entity in EntitiesBySerial.Values) { - World.PushToCache(entity); + // Previous save's length is the cost estimate; 0 (new entity or first save) takes + // the small path. Heavy entities get dedicated chunks so multi-megabyte payloads + // spread across workers instead of riding inside one shared chunk. + if (entity.SerializedLength > SerializationChunkSource.HeavyEntityThreshold) + { + World.PushSingleToCache(entity); + } + else + { + World.PushToCache(entity); + } } - - World.PushToCache(this); } private static ConstructorInfo GetConstructorFor(string typeName, Type t, Type[] constructorTypes) @@ -329,6 +344,8 @@ public class GenericEntityPersistence : GenericPersistence, IGenericEntityPer if (ctor.Invoke(ctorArgs) is T entity) { entity.Created = created; + // Cost estimate for the first save's scheduling; overwritten by every save. + entity.SerializedLength = length; entities.Add(new EntitySpan(entity, pos, length)); EntitiesBySerial[serial] = entity; } diff --git a/Projects/Server/Serialization/GenericPersistence.cs b/Projects/Server/Serialization/GenericPersistence.cs index 0afd64210..1e2af7f00 100644 --- a/Projects/Server/Serialization/GenericPersistence.cs +++ b/Projects/Server/Serialization/GenericPersistence.cs @@ -29,6 +29,11 @@ public abstract class GenericPersistence : Persistence, IGenericSerializable public int SerializedPosition { get; set; } public int SerializedLength { get; set; } + private long _loadedFileLength; + + // Scheduling estimate only: previous save's payload size, or the loaded file size before the first save. + internal long EstimatedSize => SerializedLength > 0 ? SerializedLength : _loadedFileLength; + public GenericPersistence(string name, int priority) : base(priority) { Name = name; @@ -37,7 +42,9 @@ public abstract class GenericPersistence : Persistence, IGenericSerializable public override void Serialize() { - World.PushToCache(this); + // Always a dedicated chunk: self-payloads can be arbitrarily large and must not + // ride inside a shared chunk where one worker would serialize them plus the chunk. + World.PushSingleToCache(this); } public override void WriteSnapshot(string savePath, HashSet typeSet) @@ -74,6 +81,7 @@ public abstract class GenericPersistence : Persistence, IGenericSerializable } var fileLength = file.Length; + _loadedFileLength = fileLength; string error; diff --git a/Projects/Server/Serialization/Persistence.cs b/Projects/Server/Serialization/Persistence.cs index 85fbb3c17..79ca5506e 100644 --- a/Projects/Server/Serialization/Persistence.cs +++ b/Projects/Server/Serialization/Persistence.cs @@ -113,12 +113,21 @@ public abstract class Persistence internal static void SerializeAll() { - foreach (var p in _registry) + // Largest known payloads first (LPT scheduling). An indivisible multi-megabyte system + // serialized last extends the freeze by its entire duration; serialized first it overlaps + // the entity stream. Sizes come from the previous save, or the loaded files on first save. + var ordered = new Persistence[_registry.Count]; + _registry.CopyTo(ordered); + Array.Sort(ordered, static (a, b) => GetEstimatedSize(b).CompareTo(GetEstimatedSize(a))); + + foreach (var p in ordered) { p.Serialize(); } } + private static long GetEstimatedSize(Persistence p) => (p as GenericPersistence)?.EstimatedSize ?? 0; + internal static void PostWorldSaveAll() { foreach (var p in _registry) diff --git a/Projects/Server/Serialization/SerializationChunkSource.cs b/Projects/Server/Serialization/SerializationChunkSource.cs new file mode 100644 index 000000000..0b0af73cd --- /dev/null +++ b/Projects/Server/Serialization/SerializationChunkSource.cs @@ -0,0 +1,124 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: SerializationChunkSource.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Server; + +/// +/// Single-producer/multi-consumer handoff between the game loop and the serialization +/// thread workers during a world save. The producer batches entities into pooled chunks +/// so the per-entity cost is a plain array store instead of a synchronized enqueue, and +/// workers pull whole chunks so they naturally load-balance: a worker busy with a thick +/// entity simply takes fewer chunks. +/// Entities whose previous serialized size exceeds are +/// published as dedicated single-entity chunks so multi-megabyte payloads spread across +/// workers instead of riding inside one chunk. +/// +public sealed class SerializationChunkSource +{ + // 4096 refs (32KB per chunk) keeps producer sync cost at one enqueue per 4096 entities + // while the drain tail stays sub-millisecond. + private const int ChunkCapacity = 4096; + + /// + /// Entities whose previous exceeds this + /// should be pushed with . Callers do the check where the entity's + /// concrete type is known, so the size read is not an interface dispatch per entity. + /// + public const int HeavyEntityThreshold = 1024 * 1024; // 1MB + + internal readonly struct Chunk + { + public readonly IGenericSerializable Single; + public readonly IGenericSerializable[] Buffer; + public readonly int Count; + + public Chunk(IGenericSerializable single) + { + Single = single; + Buffer = null; + Count = 1; + } + + public Chunk(IGenericSerializable[] buffer, int count) + { + Single = null; + Buffer = buffer; + Count = count; + } + } + + private readonly ConcurrentQueue _chunks = new(); + private readonly ConcurrentQueue _pool = new(); + + // Producer state - written only by the game loop thread. + private IGenericSerializable[] _current; + private int _count; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Push(IGenericSerializable entity) + { + var current = _current ??= Rent(); + + // Ref store skips the bounds and array-covariance checks. Safe by construction: + // _count is producer-thread-only and always < ChunkCapacity here (reset on publish), + // and the array's element type is exactly IGenericSerializable. + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(current), _count) = entity; + + if (++_count == ChunkCapacity) + { + _chunks.Enqueue(new Chunk(current, ChunkCapacity)); + _current = null; + _count = 0; + } + } + + /// + /// Publishes the entity as a dedicated chunk regardless of its estimated size. + /// Used for persistence self-payloads, which can be large on the first save + /// before a estimate exists. + /// + public void PushSingle(IGenericSerializable entity) => _chunks.Enqueue(new Chunk(entity)); + + /// + /// Publishes the partial chunk, if any. Must be called on the producer thread before + /// the workers are told to finish draining, or the tail of the stream is not serialized. + /// + public void Flush() + { + if (_count > 0) + { + _chunks.Enqueue(new Chunk(_current, _count)); + _current = null; + _count = 0; + } + } + + internal bool TryTake(out Chunk chunk) => _chunks.TryDequeue(out chunk); + + internal void Return(IGenericSerializable[] buffer, int count) + { + // Clear so pooled chunks don't keep entities reachable between saves. + Array.Clear(buffer, 0, count); + _pool.Enqueue(buffer); + } + + private IGenericSerializable[] Rent() => + _pool.TryDequeue(out var buffer) ? buffer : new IGenericSerializable[ChunkCapacity]; +} diff --git a/Projects/Server/Serialization/SerializationThreadWorker.cs b/Projects/Server/Serialization/SerializationThreadWorker.cs index bb0e67783..4559bfbdb 100644 --- a/Projects/Server/Serialization/SerializationThreadWorker.cs +++ b/Projects/Server/Serialization/SerializationThreadWorker.cs @@ -14,7 +14,6 @@ *************************************************************************/ using System; -using System.Collections.Concurrent; using System.Runtime.CompilerServices; using System.Threading; @@ -27,23 +26,30 @@ public class SerializationThreadWorker private readonly Thread _thread; private readonly AutoResetEvent _startEvent; // Main thread tells the thread to start working private readonly AutoResetEvent _stopEvent; // Main thread waits for the worker finish draining + private readonly SerializationChunkSource _chunkSource; + private readonly int _heapSizeHint; private bool _pause; private bool _exit; private bool _exited; private byte[] _heap; + private long _entitiesSerialized; + private long _bytesSerialized; - private readonly ConcurrentQueue _entities; - - public SerializationThreadWorker(int index) + public SerializationThreadWorker(int index, SerializationChunkSource chunkSource, int heapSizeHint = 0) { _index = index; + _chunkSource = chunkSource; + _heapSizeHint = heapSizeHint; _startEvent = new AutoResetEvent(false); _stopEvent = new AutoResetEvent(false); - _entities = new ConcurrentQueue(); _thread = new Thread(Execute); _thread.Start(this); } + // Stats from the most recent save, for diagnosing load balance. + public long EntitiesSerialized => _entitiesSerialized; + public long BytesSerialized => _bytesSerialized; + public void Wake() { _startEvent.Set(); @@ -68,43 +74,77 @@ public class SerializationThreadWorker Sleep(); } - public void AllocateHeap() => _heap ??= GC.AllocateUninitializedArray(MinHeapSize); // 1MB - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Push(IGenericSerializable entity) => _entities.Enqueue(entity); + // Sized from the previous world load so the first save doesn't pay copy-on-grow during the freeze. + public void AllocateHeap() => + _heap ??= GC.AllocateUninitializedArray(Math.Max(MinHeapSize, _heapSizeHint)); [MethodImpl(MethodImplOptions.AggressiveInlining)] public ReadOnlySpan GetHeap(int start, int length) => _heap.AsSpan(start, length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Serialize(IGenericSerializable e, BufferWriter writer, byte threadIndex) + { + e.SerializedThread = threadIndex; + var start = e.SerializedPosition = (int)writer.Position; + e.Serialize(writer); + e.SerializedLength = (int)(writer.Position - start); + } + private static void Execute(object obj) { var worker = (SerializationThreadWorker)obj; var threadIndex = (byte)worker._index; - var queue = worker._entities; + var chunkSource = worker._chunkSource; var serializedTypes = World.SerializedTypes; while (worker._startEvent.WaitOne()) { var writer = new BufferWriter(worker._heap, true, serializedTypes); + var entities = 0L; + var spinner = new SpinWait(); while (true) { var pauseRequested = Volatile.Read(ref worker._pause); - if (queue.TryDequeue(out var e)) + if (chunkSource.TryTake(out var chunk)) { - e.SerializedThread = threadIndex; - var start = e.SerializedPosition = (int)writer.Position; - e.Serialize(writer); - e.SerializedLength = (int)(writer.Position - start); + spinner.Reset(); + + if (chunk.Single != null) + { + Serialize(chunk.Single, writer, threadIndex); + entities++; + } + else + { + var buffer = chunk.Buffer; + var count = chunk.Count; + for (var i = 0; i < count; i++) + { + Serialize(buffer[i], writer, threadIndex); + } + + entities += count; + chunkSource.Return(buffer, count); + } } else if (pauseRequested) // Break when finished { break; } + else + { + // Idle backoff instead of hammering the queue head while the producer works. + // sleep1Threshold: -1 keeps escalation at Yield/Sleep(0) and never Sleep(1), + // avoiding timer-resolution stalls at the end of the drain. + spinner.SpinOnce(-1); + } } worker._heap = writer.Buffer; + worker._entitiesSerialized = entities; + worker._bytesSerialized = writer.Position; writer.Close(); diff --git a/Projects/Server/World/World.cs b/Projects/Server/World/World.cs index f25ec163b..9df761fa4 100644 --- a/Projects/Server/World/World.cs +++ b/Projects/Server/World/World.cs @@ -44,8 +44,8 @@ public static class World private static readonly MobilePersistence _mobilePersistence = new(); private static readonly GenericEntityPersistence _guildPersistence = new("Guilds", 3, 1, 0x7FFFFFFF); - private static int _threadId; internal static SerializationThreadWorker[] _threadWorkers; + private static readonly SerializationChunkSource _chunkSource = new(); private static readonly ManualResetEvent _diskWriteHandle = new(true); private static string _tempSavePath; // Path to the temporary folder for the save @@ -199,9 +199,36 @@ public static class World var threadCount = UseMultiThreadedSaves ? Math.Max(Environment.ProcessorCount - 1, 1) : 1; _threadWorkers = new SerializationThreadWorker[threadCount]; + // The save we just loaded tells us how big the heaps need to be, so the first save + // doesn't pay copy-on-grow inside the freeze window. 25% headroom for world growth. + var heapSizeHint = (int)Math.Min(GetLoadedSaveSize() / threadCount * 5 / 4, 1024 * 1024 * 1024); + for (var i = 0; i < _threadWorkers.Length; i++) { - _threadWorkers[i] = new SerializationThreadWorker(i); + _threadWorkers[i] = new SerializationThreadWorker(i, _chunkSource, heapSizeHint); + } + } + + private static long GetLoadedSaveSize() + { + try + { + if (!Directory.Exists(SavePath)) + { + return 0; + } + + var total = 0L; + foreach (var file in Directory.EnumerateFiles(SavePath, "*.bin", SearchOption.AllDirectories)) + { + total += new FileInfo(file).Length; + } + + return total; + } + catch + { + return 0; } } @@ -304,6 +331,7 @@ public static class World Persistence.SerializeAll(); PauseSerializationThreads(); + LogWorkerBalance(); EventSink.InvokeWorldSave(); } @@ -388,6 +416,33 @@ public static class World MovementThrottle.ResetAllMovementTiming(); // Prevent post-save movement rejection bursts } + private static void LogWorkerBalance() + { + var totalEntities = 0L; + var totalBytes = 0L; + var minBytes = long.MaxValue; + var maxBytes = 0L; + + for (var i = 0; i < _threadWorkers.Length; i++) + { + var worker = _threadWorkers[i]; + totalEntities += worker.EntitiesSerialized; + var bytes = worker.BytesSerialized; + totalBytes += bytes; + minBytes = Math.Min(minBytes, bytes); + maxBytes = Math.Max(maxBytes, bytes); + } + + logger.Debug( + "Serialized {EntityCount} entities ({ByteCount} bytes) across {WorkerCount} workers (min {MinBytes}, max {MaxBytes} bytes per worker)", + totalEntities, + totalBytes, + _threadWorkers.Length, + minBytes, + maxBytes + ); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void WakeSerializationThreads() { @@ -397,9 +452,11 @@ public static class World } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void PauseSerializationThreads() { + // Publish the partial chunk before the workers are told to finish draining. + _chunkSource.Flush(); + for (var i = 0; i < _threadWorkers.Length; i++) { _threadWorkers[i].Sleep(); @@ -407,20 +464,10 @@ public static class World } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int GetThreadWorkerCount() => Math.Max(Environment.ProcessorCount - 1, 1); + internal static void PushToCache(IGenericSerializable e) => _chunkSource.Push(e); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void ResetRoundRobin() => _threadId = 0; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void PushToCache(IGenericSerializable e) - { - _threadWorkers[_threadId++].Push(e); - if (_threadId == _threadWorkers.Length) - { - _threadId = 0; - } - } + internal static void PushSingleToCache(IGenericSerializable e) => _chunkSource.PushSingle(e); public static void ExitSerializationThreads() {