feat: Upgrades serialization v4 (Threaded Heap Serialization) (#1947)
### Summary - `GenericEntityPersistence` is now a type of `GenericPersistence`. This allows developers to serialize both entities and non-entities in the same system. 🎉 - Each `SerializationThreadWorker` now allocates 1MB of heap for serialization _permanently_. If more memory is needed, that thread will double it's memory, not to exceed increments of 64MB. - Several bugs with serialization introduced with the pure MMF implementation have been fixed. - `BinaryFileReader` has been added back. 🎉 - Adds `world.useMultithreadedSaves` to allow disabling threaded saves. > [!IMPORTANT] > **Developer Note** > The split file serialization has been deprecated and is no longer used. We have effectively gone back to the same file writing we had before the pure MMF implementation.
This commit is contained in:
Kamron Batman
GitHub
parent
aaac0c596c
commit
465d3c8187
20 changed files with 564 additions and 373 deletions
|
|
@ -55,8 +55,8 @@ public static class AdhocPersistence
|
|||
{
|
||||
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
|
||||
PathUtility.EnsureDirectory(Path.GetDirectoryName(fullPath));
|
||||
ConcurrentQueue<Type> types = [];
|
||||
var writer = new MemoryMapFileWriter(new FileStream(filePath, FileMode.Create), sizeHint, types);
|
||||
HashSet<Type> typesSet = [];
|
||||
var writer = new MemoryMapFileWriter(new FileStream(filePath, FileMode.Create), sizeHint, typesSet);
|
||||
serializer(writer);
|
||||
|
||||
Task.Run(
|
||||
|
|
@ -67,14 +67,6 @@ public static class AdhocPersistence
|
|||
writer.Dispose();
|
||||
fs.Dispose();
|
||||
|
||||
HashSet<Type> typesSet = [];
|
||||
|
||||
// Dedupe the queue.
|
||||
foreach (var type in types)
|
||||
{
|
||||
typesSet.Add(type);
|
||||
}
|
||||
|
||||
Persistence.WriteSerializedTypesSnapshot(Path.GetDirectoryName(fullPath), typesSet);
|
||||
},
|
||||
Core.ClosingTokenSource.Token
|
||||
|
|
|
|||
109
Projects/Server/Serialization/BinaryFileReader.cs
Normal file
109
Projects/Server/Serialization/BinaryFileReader.cs
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: BinaryFileReader.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* You should have received a copy of the GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public sealed unsafe class BinaryFileReader : IDisposable, IGenericReader
|
||||
{
|
||||
private readonly bool _usePrefixes;
|
||||
private readonly MemoryMappedFile _mmf;
|
||||
private readonly MemoryMappedViewStream _accessor;
|
||||
private readonly UnmanagedDataReader _reader;
|
||||
|
||||
public BinaryFileReader(string path, bool usePrefixes = true, Encoding encoding = null)
|
||||
{
|
||||
_usePrefixes = usePrefixes;
|
||||
var fi = new FileInfo(path);
|
||||
|
||||
if (fi.Length > 0)
|
||||
{
|
||||
_mmf = MemoryMappedFile.CreateFromFile(path, FileMode.Open);
|
||||
_accessor = _mmf.CreateViewStream();
|
||||
byte* ptr = null;
|
||||
_accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
_reader = new UnmanagedDataReader(ptr, _accessor.Length, encoding: encoding);
|
||||
}
|
||||
else
|
||||
{
|
||||
_reader = new UnmanagedDataReader(null, 0, encoding: encoding);
|
||||
}
|
||||
}
|
||||
|
||||
public long Position => _reader.Position;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_accessor?.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
_accessor?.Dispose();
|
||||
_mmf?.Dispose();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadString(bool intern = false) => _usePrefixes ? _reader.ReadString(intern) : _reader.ReadStringRaw(intern);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadStringRaw(bool intern = false) => _reader.ReadStringRaw(intern);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public long ReadLong() => _reader.ReadLong();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ulong ReadULong() => _reader.ReadULong();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int ReadInt() => _reader.ReadInt();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public uint ReadUInt() => _reader.ReadUInt();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public short ReadShort() => _reader.ReadShort();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ushort ReadUShort() => _reader.ReadUShort();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public double ReadDouble() => _reader.ReadDouble();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public float ReadFloat() => _reader.ReadFloat();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte ReadByte() => _reader.ReadByte();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public sbyte ReadSByte() => _reader.ReadSByte();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool ReadBool() => _reader.ReadBool();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Serial ReadSerial() => _reader.ReadSerial();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Type ReadType() => _reader.ReadType();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int Read(Span<byte> buffer) => _reader.Read(buffer);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public long Seek(long offset, SeekOrigin origin) => _reader.Seek(offset, origin);
|
||||
}
|
||||
|
|
@ -105,13 +105,7 @@ public class BufferWriter : IGenericWriter
|
|||
_buffer = newBuffer;
|
||||
}
|
||||
|
||||
public virtual void Flush()
|
||||
{
|
||||
// Need to avoid buffer.Length = 2, buffer * 2 is 4, but we need 8 or 16bytes, causing an exception.
|
||||
// The least we need is 16bytes + Index, but we use BufferSize since it should always be big enough for a single
|
||||
// non-dynamic field.
|
||||
Resize(Math.Max(BufferSize, _buffer.Length * 2));
|
||||
}
|
||||
public virtual void Flush() => Resize(Math.Clamp(_buffer.Length * 2, BufferSize, _buffer.Length + 1024 * 1024 * 64));
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void FlushIfNeeded(int amount)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
|
|
@ -32,141 +31,115 @@ public interface IGenericEntityPersistence
|
|||
public void DeserializeIndexes(string savePath, Dictionary<ulong, string> typesDb);
|
||||
}
|
||||
|
||||
public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistence where T : class, ISerializable
|
||||
public class GenericEntityPersistence<T> : GenericPersistence, IGenericEntityPersistence where T : class, ISerializable
|
||||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(GenericEntityPersistence<T>));
|
||||
private static List<EntitySpan<T>>[] _entities;
|
||||
|
||||
private long _initialIdxSize = 1024 * 256;
|
||||
private long _initialBinSize = 1024 * 1024;
|
||||
private readonly string _name;
|
||||
private readonly uint _minSerial;
|
||||
private readonly uint _maxSerial;
|
||||
// Support legacy split file serialization
|
||||
private static Dictionary<int, List<EntitySpan<T>>> _entities;
|
||||
|
||||
private readonly Serial _minSerial;
|
||||
private readonly Serial _maxSerial;
|
||||
private Serial _lastEntitySerial;
|
||||
private readonly Dictionary<Serial, T> _pendingAdd = new();
|
||||
private readonly Dictionary<Serial, T> _pendingDelete = new();
|
||||
|
||||
private readonly uint[] _entitiesCount = new uint[World.GetThreadWorkerCount()];
|
||||
|
||||
private readonly (MemoryMapFileWriter idxWriter, MemoryMapFileWriter binWriter)[] _writers =
|
||||
new (MemoryMapFileWriter, MemoryMapFileWriter)[World.GetThreadWorkerCount()];
|
||||
|
||||
public Dictionary<Serial, T> EntitiesBySerial { get; } = new();
|
||||
|
||||
public GenericEntityPersistence(string name, int priority, uint minSerial, uint maxSerial) : base(priority)
|
||||
public GenericEntityPersistence(string name, int priority, uint minSerial, uint maxSerial) : this(
|
||||
name,
|
||||
priority,
|
||||
(Serial)minSerial,
|
||||
(Serial)maxSerial
|
||||
)
|
||||
{
|
||||
}
|
||||
|
||||
public GenericEntityPersistence(string name, int priority, Serial minSerial, Serial maxSerial) : base(name, priority)
|
||||
{
|
||||
_name = name;
|
||||
_minSerial = minSerial;
|
||||
_maxSerial = maxSerial;
|
||||
_lastEntitySerial = (Serial)(minSerial - 1);
|
||||
_lastEntitySerial = minSerial - 1;
|
||||
typeof(T).RegisterFindEntity(Find);
|
||||
}
|
||||
|
||||
public override void Preserialize(string savePath, ConcurrentQueue<Type> types)
|
||||
public override void WriteSnapshot(string savePath, HashSet<Type> typeSet)
|
||||
{
|
||||
var path = Path.Combine(savePath, _name);
|
||||
PathUtility.EnsureDirectory(path);
|
||||
var dir = Path.Combine(savePath, Name);
|
||||
PathUtility.EnsureDirectory(dir);
|
||||
|
||||
var threadCount = World.GetThreadWorkerCount();
|
||||
for (var i = 0; i < threadCount; i++)
|
||||
var threads = World._threadWorkers;
|
||||
|
||||
using var binFs = new FileStream(Path.Combine(dir, $"{Name}.bin"), FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
using var idxFs = new FileStream(Path.Combine(dir, $"{Name}.idx"), FileMode.Create);
|
||||
using var idx = new MemoryMapFileWriter(idxFs, 1024 * 1024, typeSet); // 1MB
|
||||
|
||||
var binPosition = 0L;
|
||||
|
||||
// Support for non-entity generic serialization.
|
||||
if (SerializedLength > 0)
|
||||
{
|
||||
var idxPath = Path.Combine(path, $"{_name}_{i}.idx");
|
||||
var binPath = Path.Combine(path, $"{_name}_{i}.bin");
|
||||
|
||||
_writers[i] = (
|
||||
new MemoryMapFileWriter(new FileStream(idxPath, FileMode.Create), _initialIdxSize, types),
|
||||
new MemoryMapFileWriter(new FileStream(binPath, FileMode.Create), _initialBinSize, types)
|
||||
);
|
||||
|
||||
_writers[i].idxWriter.Write(3); // version
|
||||
_writers[i].idxWriter.Seek(4, SeekOrigin.Current); // Entity count
|
||||
|
||||
_entitiesCount[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public override void Serialize(IGenericSerializable e, int threadIndex)
|
||||
{
|
||||
var (idx, bin) = _writers[threadIndex];
|
||||
var pos = bin.Position;
|
||||
|
||||
var entity = (ISerializable)e;
|
||||
|
||||
entity.Serialize(bin);
|
||||
var length = (uint)(bin.Position - pos);
|
||||
|
||||
var t = entity.GetType();
|
||||
idx.Write(t);
|
||||
idx.Write(entity.Serial);
|
||||
idx.Write(entity.Created.Ticks);
|
||||
idx.Write(pos);
|
||||
idx.Write(length);
|
||||
|
||||
_entitiesCount[threadIndex]++;
|
||||
}
|
||||
|
||||
public override void WriteSnapshot()
|
||||
{
|
||||
var wroteFile = false;
|
||||
string folderPath = null;
|
||||
for (int i = 0; i < _writers.Length; i++)
|
||||
{
|
||||
var (idxWriter, binWriter) = _writers[i];
|
||||
|
||||
var binBytesWritten = binWriter.Position;
|
||||
|
||||
// Write the entity count
|
||||
var pos = idxWriter.Position;
|
||||
idxWriter.Seek(4, SeekOrigin.Begin);
|
||||
idxWriter.Write(_entitiesCount[i]);
|
||||
idxWriter.Seek(pos, SeekOrigin.Begin);
|
||||
|
||||
var idxFs = idxWriter.FileStream;
|
||||
var idxFilePath = idxFs.Name;
|
||||
var binFs = binWriter.FileStream;
|
||||
var binFilePath = binFs.Name;
|
||||
|
||||
if (_initialIdxSize < idxFs.Position)
|
||||
try
|
||||
{
|
||||
_initialIdxSize = idxFs.Position;
|
||||
binFs.Write(threads[SerializedThread].GetHeap(SerializedPosition, SerializedLength));
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
logger.Error(
|
||||
error,
|
||||
"Error writing entity: (Thread: {Thread} - {Start} {Length})",
|
||||
SerializedThread,
|
||||
SerializedPosition,
|
||||
SerializedLength
|
||||
);
|
||||
}
|
||||
|
||||
if (_initialBinSize < binFs.Position)
|
||||
{
|
||||
_initialBinSize = binFs.Position;
|
||||
}
|
||||
|
||||
idxWriter.Dispose();
|
||||
binWriter.Dispose();
|
||||
|
||||
idxFs.Dispose();
|
||||
binFs.Dispose();
|
||||
|
||||
if (binBytesWritten > 1)
|
||||
{
|
||||
wroteFile = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
File.Delete(idxFilePath);
|
||||
File.Delete(binFilePath);
|
||||
folderPath = Path.GetDirectoryName(idxFilePath);
|
||||
}
|
||||
binPosition += SerializedLength;
|
||||
}
|
||||
|
||||
if (!wroteFile && folderPath != null)
|
||||
idx.Write(3); // Version
|
||||
idx.Write(EntitiesBySerial.Values.Count);
|
||||
|
||||
foreach (var e in EntitiesBySerial.Values)
|
||||
{
|
||||
Directory.Delete(folderPath);
|
||||
var thread = e.SerializedThread;
|
||||
var heapStart = e.SerializedPosition;
|
||||
var heapLength = e.SerializedLength;
|
||||
|
||||
idx.Write(e.GetType());
|
||||
idx.Write(e.Serial);
|
||||
idx.Write(e.Created.Ticks);
|
||||
idx.Write(binPosition);
|
||||
idx.Write(heapLength);
|
||||
|
||||
try
|
||||
{
|
||||
binFs.Write(threads[thread].GetHeap(heapStart, heapLength));
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
logger.Error(
|
||||
error,
|
||||
"Error writing entity: {Entity} (Thread: {Thread} - {Start} {Length})",
|
||||
e,
|
||||
thread,
|
||||
heapStart,
|
||||
heapLength
|
||||
);
|
||||
}
|
||||
|
||||
binPosition += heapLength;
|
||||
}
|
||||
}
|
||||
|
||||
public override void Serialize()
|
||||
{
|
||||
World.ResetRoundRobin();
|
||||
foreach (var entity in EntitiesBySerial.Values)
|
||||
{
|
||||
World.PushToCache((entity, this));
|
||||
World.PushToCache(entity);
|
||||
}
|
||||
|
||||
World.PushToCache(this);
|
||||
}
|
||||
|
||||
private static ConstructorInfo GetConstructorFor(string typeName, Type t, Type[] constructorTypes)
|
||||
|
|
@ -205,7 +178,7 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
*/
|
||||
private unsafe Dictionary<int, ConstructorInfo> ReadTypes(string savePath)
|
||||
{
|
||||
string typesPath = Path.Combine(savePath, _name, $"{_name}.tdb");
|
||||
string typesPath = Path.Combine(savePath, Name, $"{Name}.tdb");
|
||||
if (!File.Exists(typesPath))
|
||||
{
|
||||
return null;
|
||||
|
|
@ -236,59 +209,57 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
|
||||
public virtual void DeserializeIndexes(string savePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
string indexPath = Path.Combine(savePath, _name, $"{_name}.idx");
|
||||
string indexPath = Path.Combine(savePath, Name, $"{Name}.idx");
|
||||
|
||||
_entities ??= [];
|
||||
|
||||
// Support for legacy MUO Serialization that used split files
|
||||
if (!File.Exists(indexPath))
|
||||
{
|
||||
TryDeserializeMultithreadIndexes(savePath, typesDb);
|
||||
TryDeserializeSplitFileIndexes(savePath, typesDb);
|
||||
return;
|
||||
}
|
||||
|
||||
_entities = [InternalDeserializeIndexes(indexPath, typesDb)];
|
||||
InternalDeserializeIndexes(indexPath, typesDb, _entities[0] = []);
|
||||
}
|
||||
|
||||
private void TryDeserializeMultithreadIndexes(string savePath, Dictionary<ulong, string> typesDb)
|
||||
private void TryDeserializeSplitFileIndexes(string savePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
var index = 0;
|
||||
var fileList = new List<string>();
|
||||
while (true)
|
||||
{
|
||||
var path = Path.Combine(savePath, _name, $"{_name}_{index}.idx");
|
||||
var path = Path.Combine(savePath, Name, $"{Name}_{index}.idx");
|
||||
var fi = new FileInfo(path);
|
||||
if (!fi.Exists)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (fi.Length != 0)
|
||||
if (fi.Length == 0)
|
||||
{
|
||||
fileList.Add(path);
|
||||
continue;
|
||||
}
|
||||
|
||||
InternalDeserializeIndexes(path, typesDb, _entities[index] = []);
|
||||
index++;
|
||||
}
|
||||
|
||||
_entities = new List<EntitySpan<T>>[fileList.Count];
|
||||
for (var i = 0; i < fileList.Count; i++)
|
||||
{
|
||||
_entities[i] = InternalDeserializeIndexes(fileList[i], typesDb);
|
||||
}
|
||||
}
|
||||
|
||||
private unsafe List<EntitySpan<T>> InternalDeserializeIndexes(string filePath, Dictionary<ulong, string> typesDb)
|
||||
private unsafe void InternalDeserializeIndexes(
|
||||
string filePath, Dictionary<ulong, string> typesDb, List<EntitySpan<T>> entities
|
||||
)
|
||||
{
|
||||
object[] ctorArgs = new object[1];
|
||||
List<EntitySpan<T>> entities = [];
|
||||
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(filePath, FileMode.Open);
|
||||
using var accessor = mmf.CreateViewStream();
|
||||
|
||||
byte* ptr = null;
|
||||
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
UnmanagedDataReader dataReader = new UnmanagedDataReader(ptr, accessor.Length, typesDb);
|
||||
var dataReader = new UnmanagedDataReader(ptr, accessor.Length);
|
||||
|
||||
var version = dataReader.ReadInt();
|
||||
|
||||
Dictionary<int, ConstructorInfo> ctors = null;
|
||||
Dictionary<int, ConstructorInfo> ctors = [];
|
||||
|
||||
if (version < 2)
|
||||
{
|
||||
ctors = ReadTypes(Path.GetDirectoryName(filePath));
|
||||
|
|
@ -296,15 +267,16 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
|
||||
if (typesDb == null && ctors == null)
|
||||
{
|
||||
return entities;
|
||||
return;
|
||||
}
|
||||
|
||||
int count = dataReader.ReadInt();
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var ctorArgs = new object[1];
|
||||
Type[] ctorArguments = [typeof(Serial)];
|
||||
|
||||
for (int i = 0; i < count; ++i)
|
||||
var count = dataReader.ReadInt();
|
||||
|
||||
for (var i = 0; i < count; ++i)
|
||||
{
|
||||
ConstructorInfo ctor;
|
||||
// Version 2 & 3 with SerializedTypes.db
|
||||
|
|
@ -331,6 +303,7 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
{
|
||||
dataReader.ReadLong(); // LastSerialized
|
||||
}
|
||||
|
||||
var pos = dataReader.ReadLong();
|
||||
var length = dataReader.ReadInt();
|
||||
|
||||
|
|
@ -344,45 +317,39 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
if (ctor.Invoke(ctorArgs) is T entity)
|
||||
{
|
||||
entity.Created = created;
|
||||
entities.Add(new EntitySpan<T>(entity, pos, (int)length));
|
||||
entities.Add(new EntitySpan<T>(entity, pos, length));
|
||||
EntitiesBySerial[serial] = entity;
|
||||
}
|
||||
}
|
||||
|
||||
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
entities.TrimExcess();
|
||||
|
||||
if (EntitiesBySerial.Count > 0)
|
||||
{
|
||||
_lastEntitySerial = EntitiesBySerial.Keys.Max();
|
||||
}
|
||||
|
||||
return entities;
|
||||
}
|
||||
|
||||
public override void Deserialize(string savePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
string dataPath = Path.Combine(savePath, _name, $"{_name}.bin");
|
||||
string dataPath = Path.Combine(savePath, Name, $"{Name}.bin");
|
||||
var fi = new FileInfo(dataPath);
|
||||
|
||||
if (!fi.Exists)
|
||||
{
|
||||
TryDeserializeMultithread(savePath, typesDb);
|
||||
}
|
||||
else
|
||||
else if (fi.Length > 0)
|
||||
{
|
||||
if (fi.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
InternalDeserialize(dataPath, 0, typesDb);
|
||||
}
|
||||
|
||||
_entities.Clear();
|
||||
_entities.TrimExcess();
|
||||
_entities = null;
|
||||
}
|
||||
|
||||
private static unsafe void InternalDeserialize(string filePath, int index, Dictionary<ulong, string> typesDb)
|
||||
private unsafe void InternalDeserialize(string filePath, int index, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(filePath, FileMode.Open);
|
||||
using var accessor = mmf.CreateViewStream();
|
||||
|
|
@ -390,6 +357,9 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
byte* ptr = null;
|
||||
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
UnmanagedDataReader dataReader = new UnmanagedDataReader(ptr, accessor.Length, typesDb);
|
||||
|
||||
Deserialize(dataReader);
|
||||
|
||||
var deleteAllFailures = false;
|
||||
|
||||
foreach (var entry in _entities[index])
|
||||
|
|
@ -460,15 +430,25 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
return;
|
||||
}
|
||||
|
||||
var folderPath = Path.Combine(savePath, _name);
|
||||
var folderPath = Path.Combine(savePath, Name);
|
||||
|
||||
for (var i = 0; i < _entities.Length; i++)
|
||||
foreach (var i in _entities.Keys)
|
||||
{
|
||||
var path = Path.Combine(folderPath, $"{_name}_{i}.bin");
|
||||
var path = Path.Combine(folderPath, $"{Name}_{i}.bin");
|
||||
InternalDeserialize(path, i, typesDb);
|
||||
}
|
||||
}
|
||||
|
||||
// Override for non-entity serialization
|
||||
public override void Serialize(IGenericWriter writer)
|
||||
{
|
||||
}
|
||||
|
||||
// Override for non-entity deserialization
|
||||
public override void Deserialize(IGenericReader reader)
|
||||
{
|
||||
}
|
||||
|
||||
public override void PostWorldSave()
|
||||
{
|
||||
ProcessSafetyQueues();
|
||||
|
|
@ -492,25 +472,26 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
);
|
||||
}
|
||||
#endif
|
||||
var last = _lastEntitySerial;
|
||||
var max = (Serial)_maxSerial;
|
||||
var last = (uint)_lastEntitySerial;
|
||||
var min = (uint)_minSerial;
|
||||
var max = (uint)_maxSerial;
|
||||
|
||||
for (uint i = 0; i < _maxSerial; i++)
|
||||
for (uint i = 0; i < max; i++)
|
||||
{
|
||||
last++;
|
||||
|
||||
if (last > max)
|
||||
{
|
||||
last = (Serial)_minSerial;
|
||||
last = min;
|
||||
}
|
||||
|
||||
if (FindEntity<T>(last) == null)
|
||||
if (FindEntity<T>((Serial)last) == null)
|
||||
{
|
||||
return _lastEntitySerial = last;
|
||||
return _lastEntitySerial = (Serial)last;
|
||||
}
|
||||
}
|
||||
|
||||
OutOfMemory($"No serials left to allocate for {_name}");
|
||||
OutOfMemory($"No serials left to allocate for {Name}");
|
||||
return Serial.MinusOne;
|
||||
}
|
||||
}
|
||||
|
|
@ -530,17 +511,21 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
goto case WorldState.Loading;
|
||||
}
|
||||
case WorldState.Loading:
|
||||
case WorldState.WritingSave:
|
||||
{
|
||||
if (_pendingDelete.Remove(entity.Serial))
|
||||
{
|
||||
logger.Warning("Deleted then added {Entity} during {WorldState} state.", entity.GetType().Name, worldState.ToString());
|
||||
logger.Warning(
|
||||
"Deleted then added {Entity} during {WorldState} state.",
|
||||
entity.GetType().Name,
|
||||
worldState.ToString()
|
||||
);
|
||||
}
|
||||
|
||||
_pendingAdd[entity.Serial] = entity;
|
||||
break;
|
||||
}
|
||||
case WorldState.PendingSave:
|
||||
case WorldState.WritingSave:
|
||||
case WorldState.Running:
|
||||
{
|
||||
ref var entityEntry = ref CollectionsMarshal.GetValueRefOrAddDefault(EntitiesBySerial, entity.Serial, out bool exists);
|
||||
|
|
@ -591,13 +576,13 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
goto case WorldState.Loading;
|
||||
}
|
||||
case WorldState.Loading:
|
||||
case WorldState.WritingSave:
|
||||
{
|
||||
_pendingAdd.Remove(entity.Serial);
|
||||
_pendingDelete[entity.Serial] = entity;
|
||||
break;
|
||||
}
|
||||
case WorldState.PendingSave:
|
||||
case WorldState.WritingSave:
|
||||
case WorldState.Running:
|
||||
{
|
||||
EntitiesBySerial.Remove(entity.Serial);
|
||||
|
|
@ -613,8 +598,6 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
AddEntity(entity);
|
||||
}
|
||||
|
||||
_pendingAdd.Clear();
|
||||
|
||||
foreach (var entity in _pendingDelete.Values)
|
||||
{
|
||||
if (_pendingAdd.ContainsKey(entity.Serial))
|
||||
|
|
@ -625,10 +608,11 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
RemoveEntity(entity);
|
||||
}
|
||||
|
||||
_pendingAdd.Clear();
|
||||
_pendingDelete.Clear();
|
||||
}
|
||||
|
||||
private void AppendSafetyLog(string action, ISerializable entity)
|
||||
private static void AppendSafetyLog(string action, ISerializable entity)
|
||||
{
|
||||
var message =
|
||||
$"Warning: Attempted to {{Action}} {{Entity}} during world save.{Environment.NewLine}This action could cause inconsistent state.{Environment.NewLine}It is strongly advised that the offending scripts be corrected.";
|
||||
|
|
@ -638,7 +622,7 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
try
|
||||
{
|
||||
using var op = new StreamWriter("world-save-errors.log", true);
|
||||
op.WriteLine("{0}\t{1}", DateTime.UtcNow, message);
|
||||
op.WriteLine($"{DateTime.UtcNow}\t{message}");
|
||||
op.WriteLine(new StackTrace(2).ToString());
|
||||
op.WriteLine();
|
||||
}
|
||||
|
|
@ -649,17 +633,9 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public T Find(Serial serial) => FindEntity<T>(serial, false, false);
|
||||
public T Find(Serial serial, bool returnDeleted = false) => FindEntity<T>(serial, returnDeleted);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public T Find(Serial serial, bool returnDeleted) => FindEntity<T>(serial, returnDeleted, false);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public T Find(Serial serial, bool returnDeleted, bool returnPending) => FindEntity<T>(serial, returnDeleted, returnPending);
|
||||
|
||||
public R FindEntity<R>(Serial serial) where R : class, T => FindEntity<R>(serial, false, false);
|
||||
|
||||
public R FindEntity<R>(Serial serial, bool returnDeleted, bool returnPending) where R : class, T
|
||||
public R FindEntity<R>(Serial serial, bool returnDeleted = false) where R : class, T
|
||||
{
|
||||
switch (World.WorldState)
|
||||
{
|
||||
|
|
@ -669,14 +645,14 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
}
|
||||
case WorldState.Loading:
|
||||
case WorldState.Saving:
|
||||
case WorldState.WritingSave:
|
||||
{
|
||||
if (returnDeleted && returnPending && _pendingDelete.TryGetValue(serial, out var entity))
|
||||
if (returnDeleted && _pendingDelete.TryGetValue(serial, out var entity))
|
||||
{
|
||||
return entity as R;
|
||||
}
|
||||
|
||||
if (returnPending && _pendingAdd.TryGetValue(serial, out entity) ||
|
||||
EntitiesBySerial.TryGetValue(serial, out entity))
|
||||
if (_pendingAdd.TryGetValue(serial, out entity) || EntitiesBySerial.TryGetValue(serial, out entity))
|
||||
{
|
||||
return entity as R;
|
||||
}
|
||||
|
|
@ -684,7 +660,6 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
return null;
|
||||
}
|
||||
case WorldState.PendingSave:
|
||||
case WorldState.WritingSave:
|
||||
case WorldState.Running:
|
||||
{
|
||||
return EntitiesBySerial.TryGetValue(serial, out var entity) ? entity as R : null;
|
||||
|
|
|
|||
|
|
@ -14,65 +14,105 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public abstract class GenericPersistence : Persistence, IGenericSerializable
|
||||
{
|
||||
private long _initialSize = 1024 * 1024;
|
||||
private MemoryMapFileWriter _fileToSave;
|
||||
|
||||
public string Name { get; }
|
||||
public string SaveFilePath { get; protected set; } // "<Folder>/<System>.bin"
|
||||
|
||||
public GenericPersistence(string name, int priority) : base(priority) => Name = name;
|
||||
public byte SerializedThread { get; set; }
|
||||
public int SerializedPosition { get; set; }
|
||||
public int SerializedLength { get; set; }
|
||||
|
||||
public override void Preserialize(string savePath, ConcurrentQueue<Type> types)
|
||||
public GenericPersistence(string name, int priority) : base(priority)
|
||||
{
|
||||
var path = Path.Combine(savePath, Name);
|
||||
var filePath = Path.Combine(path, $"{Name}.bin");
|
||||
PathUtility.EnsureDirectory(path);
|
||||
|
||||
_fileToSave = new MemoryMapFileWriter(new FileStream(filePath, FileMode.Create), _initialSize, types);
|
||||
Name = name;
|
||||
SaveFilePath = Path.Combine(Name, $"{Name}.bin");
|
||||
}
|
||||
|
||||
public override void Serialize()
|
||||
{
|
||||
World.ResetRoundRobin();
|
||||
World.PushToCache((this, this));
|
||||
World.PushToCache(this);
|
||||
}
|
||||
|
||||
public override void WriteSnapshot()
|
||||
public override void WriteSnapshot(string savePath, HashSet<Type> typeSet)
|
||||
{
|
||||
string folderPath = null;
|
||||
using (var fs = _fileToSave.FileStream)
|
||||
if (SerializedLength == 0)
|
||||
{
|
||||
if (fs.Position > _initialSize)
|
||||
{
|
||||
_initialSize = fs.Position;
|
||||
}
|
||||
|
||||
_fileToSave.Dispose();
|
||||
if (_fileToSave.Position == 0)
|
||||
{
|
||||
folderPath = Path.GetDirectoryName(fs.Name);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (folderPath != null)
|
||||
{
|
||||
Directory.Delete(folderPath);
|
||||
}
|
||||
var file = Path.Combine(savePath, SaveFilePath);
|
||||
var dir = Path.GetDirectoryName(file);
|
||||
PathUtility.EnsureDirectory(dir);
|
||||
|
||||
var threads = World._threadWorkers;
|
||||
|
||||
using var binFs = new FileStream(file, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
|
||||
var thread = SerializedThread;
|
||||
var heapStart = SerializedPosition;
|
||||
var heapLength = SerializedLength;
|
||||
|
||||
binFs.Write(threads[thread].GetHeap(heapStart, heapLength));
|
||||
}
|
||||
|
||||
public override void Serialize(IGenericSerializable e, int threadIndex) => Serialize(_fileToSave);
|
||||
public override unsafe void Deserialize(string savePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
// Assume savePath has the Core.BaseDirectory already prepended
|
||||
var dataPath = Path.GetFullPath(SaveFilePath, savePath);
|
||||
var file = new FileInfo(dataPath);
|
||||
|
||||
if (!file.Exists || file.Length <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var fileLength = file.Length;
|
||||
|
||||
string error;
|
||||
|
||||
try
|
||||
{
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(dataPath, FileMode.Open);
|
||||
using var accessor = mmf.CreateViewStream();
|
||||
|
||||
byte* ptr = null;
|
||||
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
var dataReader = new UnmanagedDataReader(ptr, accessor.Length, typesDb);
|
||||
Deserialize(dataReader);
|
||||
|
||||
error = dataReader.Position != fileLength
|
||||
? $"Serialized {fileLength} bytes, but {dataReader.Position} bytes deserialized"
|
||||
: null;
|
||||
|
||||
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
error = e.ToString();
|
||||
}
|
||||
|
||||
if (error != null)
|
||||
{
|
||||
Console.WriteLine($"***** Bad deserialize of {file.FullName} *****");
|
||||
Console.WriteLine(error);
|
||||
|
||||
Console.Write("Skip this file and continue? (y/n): ");
|
||||
var y = Console.ReadLine();
|
||||
|
||||
if (!y.InsensitiveEquals("y"))
|
||||
{
|
||||
throw new Exception("Deserialization failed.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public abstract void Serialize(IGenericWriter writer);
|
||||
|
||||
public override void Deserialize(string savePath, Dictionary<ulong, string> typesDb) =>
|
||||
AdhocPersistence.Deserialize(Path.Combine(savePath, Name, $"{Name}.bin"), Deserialize);
|
||||
|
||||
public abstract void Deserialize(IGenericReader reader);
|
||||
}
|
||||
|
|
|
|||
25
Projects/Server/Serialization/IGenericSerializable.cs
Normal file
25
Projects/Server/Serialization/IGenericSerializable.cs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: IGenericSerializable.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* You should have received a copy of the GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
*************************************************************************/
|
||||
|
||||
namespace Server;
|
||||
|
||||
public interface IGenericSerializable
|
||||
{
|
||||
byte SerializedThread { get; set; }
|
||||
int SerializedPosition { get; set; }
|
||||
int SerializedLength { get; set; }
|
||||
|
||||
void Serialize(IGenericWriter writer);
|
||||
}
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
|
@ -29,7 +29,7 @@ public unsafe class MemoryMapFileWriter : IGenericWriter, IDisposable
|
|||
{
|
||||
private readonly Encoding _encoding;
|
||||
|
||||
private readonly ConcurrentQueue<Type> _types;
|
||||
private readonly HashSet<Type> _types;
|
||||
private readonly FileStream _fileStream;
|
||||
private MemoryMappedFile _mmf;
|
||||
private MemoryMappedViewAccessor _accessor;
|
||||
|
|
@ -37,7 +37,7 @@ public unsafe class MemoryMapFileWriter : IGenericWriter, IDisposable
|
|||
private long _position;
|
||||
private long _size;
|
||||
|
||||
public MemoryMapFileWriter(FileStream fileStream, long initialSize, ConcurrentQueue<Type> types = null)
|
||||
public MemoryMapFileWriter(FileStream fileStream, long initialSize, HashSet<Type> types = null)
|
||||
{
|
||||
_types = types;
|
||||
_fileStream = fileStream;
|
||||
|
|
@ -244,7 +244,7 @@ public unsafe class MemoryMapFileWriter : IGenericWriter, IDisposable
|
|||
{
|
||||
Write((byte)0x2); // xxHash3 64bit
|
||||
Write(AssemblyHandler.GetTypeHash(type));
|
||||
_types.Enqueue(type);
|
||||
_types.Add(type);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
|
|
@ -53,7 +52,7 @@ public abstract class Persistence
|
|||
}
|
||||
}
|
||||
|
||||
private unsafe static Dictionary<ulong, string> LoadTypes(string path)
|
||||
private static unsafe Dictionary<ulong, string> LoadTypes(string path)
|
||||
{
|
||||
var db = new Dictionary<ulong, string>();
|
||||
|
||||
|
|
@ -85,32 +84,31 @@ public abstract class Persistence
|
|||
}
|
||||
|
||||
// Note: This is strictly on a background thread
|
||||
internal static void PreSerializeAll(string path, ConcurrentQueue<Type> types)
|
||||
internal static void WriteSnapshotAll(string path, HashSet<Type> typeSet)
|
||||
{
|
||||
foreach (var p in _registry)
|
||||
{
|
||||
p.Preserialize(path, types);
|
||||
p.WriteSnapshot(path, typeSet);
|
||||
}
|
||||
|
||||
WriteSerializedTypesSnapshot(path, typeSet);
|
||||
}
|
||||
|
||||
private static readonly HashSet<Type> _typesSet = [];
|
||||
|
||||
// Note: This is strictly on a background thread
|
||||
internal static void WriteSnapshotAll(string path, ConcurrentQueue<Type> types)
|
||||
public static void WriteSerializedTypesSnapshot(string path, HashSet<Type> types)
|
||||
{
|
||||
foreach (var p in _registry)
|
||||
{
|
||||
p.WriteSnapshot();
|
||||
}
|
||||
string typesPath = Path.Combine(path, "SerializedTypes.db");
|
||||
using var fs = new FileStream(typesPath, FileMode.Create);
|
||||
using var writer = new MemoryMapFileWriter(fs, 1024 * 1024 * 4);
|
||||
|
||||
writer.Write(0); // version
|
||||
writer.Write(types.Count);
|
||||
|
||||
// Dedupe the queue.
|
||||
foreach (var type in types)
|
||||
{
|
||||
_typesSet.Add(type);
|
||||
var fullName = type.FullName;
|
||||
writer.Write(HashUtility.ComputeHash64(fullName));
|
||||
writer.WriteStringRaw(fullName);
|
||||
}
|
||||
|
||||
WriteSerializedTypesSnapshot(path, _typesSet);
|
||||
_typesSet.Clear();
|
||||
}
|
||||
|
||||
internal static void SerializeAll()
|
||||
|
|
@ -137,32 +135,8 @@ public abstract class Persistence
|
|||
}
|
||||
}
|
||||
|
||||
public static void WriteSerializedTypesSnapshot(string path, HashSet<Type> types)
|
||||
{
|
||||
string typesPath = Path.Combine(path, "SerializedTypes.db");
|
||||
using var fs = new FileStream(typesPath, FileMode.Create);
|
||||
using var writer = new MemoryMapFileWriter(fs, 1024 * 1024 * 4);
|
||||
|
||||
writer.Write(0); // version
|
||||
writer.Write(types.Count);
|
||||
|
||||
foreach (var type in types)
|
||||
{
|
||||
var fullName = type.FullName;
|
||||
writer.Write(HashUtility.ComputeHash64(fullName));
|
||||
writer.WriteStringRaw(fullName);
|
||||
}
|
||||
}
|
||||
|
||||
// Open file streams, MMFs, prepare data structures
|
||||
// Note: This should only be run on a background thread
|
||||
public abstract void Preserialize(string savePath, ConcurrentQueue<Type> types);
|
||||
|
||||
// Note: This should only be run on a background thread
|
||||
public abstract void Serialize(IGenericSerializable e, int threadIndex);
|
||||
|
||||
// Note: This should only be run on a background thread
|
||||
public abstract void WriteSnapshot();
|
||||
public abstract void WriteSnapshot(string savePath, HashSet<Type> typeSet);
|
||||
|
||||
public abstract void Serialize();
|
||||
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@ namespace Server;
|
|||
|
||||
public static class SerializationExtensions
|
||||
{
|
||||
private static readonly Dictionary<Type, Func<Serial, bool, bool, ISerializable>> _directFinderTable = new();
|
||||
private static readonly Dictionary<Type, Func<Serial, bool, bool, ISerializable>> _searchTable = new();
|
||||
private static readonly Dictionary<Type, Func<Serial, bool, ISerializable>> _directFinderTable = new();
|
||||
private static readonly Dictionary<Type, Func<Serial, bool, ISerializable>> _searchTable = new();
|
||||
|
||||
public static void RegisterFindEntity(this Type type, Func<Serial, bool, bool, ISerializable> func)
|
||||
public static void RegisterFindEntity(this Type type, Func<Serial, bool, ISerializable> func)
|
||||
{
|
||||
_searchTable[type] = func;
|
||||
}
|
||||
|
|
@ -47,12 +47,12 @@ public static class SerializationExtensions
|
|||
|
||||
if (typeof(IEntity).IsAssignableFrom(typeT))
|
||||
{
|
||||
return World.FindEntity<IEntity>(serial, returnPending: false) as T;
|
||||
return World.FindEntity<IEntity>(serial) as T;
|
||||
}
|
||||
|
||||
if (_directFinderTable.TryGetValue(typeT, out var finder))
|
||||
{
|
||||
return finder(serial, false, false) as T;
|
||||
return finder(serial, false) as T;
|
||||
}
|
||||
|
||||
Type type = null;
|
||||
|
|
@ -87,7 +87,7 @@ public static class SerializationExtensions
|
|||
|
||||
finder = _searchTable[type];
|
||||
_directFinderTable[type] = finder;
|
||||
return finder(serial, false, false) as T;
|
||||
return finder(serial, false) as T;
|
||||
}
|
||||
|
||||
public static List<T> ReadEntityList<T>(
|
||||
|
|
|
|||
113
Projects/Server/Serialization/SerializationThreadWorker.cs
Normal file
113
Projects/Server/Serialization/SerializationThreadWorker.cs
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: SerializationThreadWorker.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* You should have received a copy of the GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public class SerializationThreadWorker
|
||||
{
|
||||
private const int MinHeapSize = 1024 * 1024; // 1MB
|
||||
private readonly int _index;
|
||||
private readonly Thread _thread;
|
||||
private readonly AutoResetEvent _startEvent; // Main thread tells the thread to start working
|
||||
private readonly AutoResetEvent _stopEvent; // Main thread waits for the worker finish draining
|
||||
private bool _pause;
|
||||
private bool _exit;
|
||||
private byte[] _heap;
|
||||
|
||||
private readonly ConcurrentQueue<IGenericSerializable> _entities;
|
||||
|
||||
public SerializationThreadWorker(int index)
|
||||
{
|
||||
_index = index;
|
||||
_startEvent = new AutoResetEvent(false);
|
||||
_stopEvent = new AutoResetEvent(false);
|
||||
_entities = new ConcurrentQueue<IGenericSerializable>();
|
||||
_thread = new Thread(Execute);
|
||||
_thread.Start(this);
|
||||
}
|
||||
|
||||
public void Wake()
|
||||
{
|
||||
_startEvent.Set();
|
||||
}
|
||||
|
||||
public void Sleep()
|
||||
{
|
||||
Volatile.Write(ref _pause, true);
|
||||
_stopEvent.WaitOne();
|
||||
}
|
||||
|
||||
public void Exit()
|
||||
{
|
||||
_exit = true;
|
||||
Wake();
|
||||
Sleep();
|
||||
}
|
||||
|
||||
public void AllocateHeap() => _heap ??= GC.AllocateUninitializedArray<byte>(MinHeapSize); // 1MB
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Push(IGenericSerializable entity) => _entities.Enqueue(entity);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ReadOnlySpan<byte> GetHeap(int start, int length) => _heap.AsSpan(start, length);
|
||||
|
||||
private static void Execute(object obj)
|
||||
{
|
||||
var worker = (SerializationThreadWorker)obj;
|
||||
var threadIndex = (byte)worker._index;
|
||||
|
||||
var queue = worker._entities;
|
||||
var serializedTypes = World.SerializedTypes;
|
||||
|
||||
while (worker._startEvent.WaitOne())
|
||||
{
|
||||
var writer = new BufferWriter(worker._heap, true, serializedTypes);
|
||||
|
||||
while (true)
|
||||
{
|
||||
var pauseRequested = Volatile.Read(ref worker._pause);
|
||||
if (queue.TryDequeue(out var e))
|
||||
{
|
||||
e.SerializedThread = threadIndex;
|
||||
var start = e.SerializedPosition = (int)writer.Position;
|
||||
e.Serialize(writer);
|
||||
e.SerializedLength = (int)(writer.Position - start);
|
||||
}
|
||||
else if (pauseRequested) // Break when finished
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
worker._heap = writer.Buffer;
|
||||
|
||||
writer.Close();
|
||||
|
||||
worker._stopEvent.Set(); // Allow the main thread to continue now that we are finished
|
||||
worker._pause = false;
|
||||
|
||||
if (Core.Closing || worker._exit)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue