feat: Updates serialization to use MMF (considerable memory savings) (#1841)
### Summary Updates the serialization strategy to use `MemoryMappedFile` instead of thick buffers. This has the benefit of being on-par with the current implementation (based on hardware/OS), however won't incur the double-memory issue. > [!Important] > **Developer Note** > The `BinaryFileWriter` and `BinaryFileReader` has been removed in favor of `MemoryMapFileWriter` and `UnmanagedDataReader`
This commit is contained in:
Kamron Batman
GitHub
parent
3b84c8052c
commit
8ec203f387
27 changed files with 1271 additions and 941 deletions
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: AdhocPersistence.cs *
|
||||
* *
|
||||
|
|
@ -25,11 +25,9 @@ namespace Server;
|
|||
public static class AdhocPersistence
|
||||
{
|
||||
/**
|
||||
* Serializes to memory synchronously. Optional buffer can be provided.
|
||||
* Note: The buffer may not be the same after returning from the function if more data is written
|
||||
* than the initial buffer can handle.
|
||||
* Serializes to memory.
|
||||
*/
|
||||
public static BufferWriter Serialize(Action<IGenericWriter> serializer, ConcurrentQueue<Type> types)
|
||||
public static IGenericWriter SerializeToBuffer(Action<IGenericWriter> serializer, ConcurrentQueue<Type> types = null)
|
||||
{
|
||||
var saveBuffer = new BufferWriter(true, types);
|
||||
serializer(saveBuffer);
|
||||
|
|
@ -37,36 +35,39 @@ public static class AdhocPersistence
|
|||
}
|
||||
|
||||
/**
|
||||
* Writes a buffer to disk. This function should be called asynchronously.
|
||||
* Writes the filePath for the binary data, and an accompanying SerializedTypes.db file of all possible types.
|
||||
* Deserializes from a buffer.
|
||||
*/
|
||||
public static void WriteSnapshot(FileInfo file, Span<byte> buffer)
|
||||
public static IGenericReader DeserializeFromBuffer(
|
||||
byte[] buffer, Action<IGenericReader> deserializer, Dictionary<ulong, string> typesDb = null
|
||||
)
|
||||
{
|
||||
var dirPath = file.DirectoryName;
|
||||
PathUtility.EnsureDirectory(dirPath);
|
||||
|
||||
using var fs = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
|
||||
fs.Write(buffer);
|
||||
var reader = new BufferReader(buffer, typesDb);
|
||||
deserializer(reader);
|
||||
return reader;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes to a memory buffer synchronously, then flushes to the path asynchronously.
|
||||
* See WriteSnapshot for more info about how to snapshot.
|
||||
* Serializes to a Memory Mapped file synchronously, then flushes to the file asynchronously.
|
||||
*/
|
||||
public static void SerializeAndSnapshot(string filePath, Action<IGenericWriter> serializer, ConcurrentQueue<Type> types = null)
|
||||
public static void SerializeAndSnapshot(
|
||||
string filePath, Action<IGenericWriter> serializer, long sizeHint = 1024 * 1024 * 32
|
||||
)
|
||||
{
|
||||
types ??= new ConcurrentQueue<Type>();
|
||||
var saveBuffer = Serialize(serializer, types);
|
||||
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
|
||||
PathUtility.EnsureDirectory(Path.GetDirectoryName(fullPath));
|
||||
ConcurrentQueue<Type> types = [];
|
||||
var writer = new MemoryMapFileWriter(new FileStream(filePath, FileMode.Create), sizeHint, types);
|
||||
serializer(writer);
|
||||
|
||||
Task.Run(
|
||||
() =>
|
||||
{
|
||||
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
|
||||
var file = new FileInfo(fullPath);
|
||||
var fs = writer.FileStream;
|
||||
|
||||
WriteSnapshot(file, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position));
|
||||
writer.Dispose();
|
||||
fs.Dispose();
|
||||
|
||||
// TODO: Create a PooledHashSet if performance becomes an issue.
|
||||
var typesSet = new HashSet<Type>();
|
||||
HashSet<Type> typesSet = [];
|
||||
|
||||
// Dedupe the queue.
|
||||
foreach (var type in types)
|
||||
|
|
@ -74,38 +75,41 @@ public static class AdhocPersistence
|
|||
typesSet.Add(type);
|
||||
}
|
||||
|
||||
Persistence.WriteSerializedTypesSnapshot(file.DirectoryName, typesSet);
|
||||
});
|
||||
Persistence.WriteSerializedTypesSnapshot(Path.GetDirectoryName(fullPath), typesSet);
|
||||
},
|
||||
Core.ClosingTokenSource.Token
|
||||
);
|
||||
}
|
||||
|
||||
public static void Deserialize(string filePath, Action<IGenericReader> deserializer)
|
||||
public static unsafe void Deserialize(string filePath, Action<IGenericReader> deserializer)
|
||||
{
|
||||
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
|
||||
var file = new FileInfo(fullPath);
|
||||
|
||||
if (!file.Exists)
|
||||
if (!file.Exists || file.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var fileLength = file.Length;
|
||||
if (fileLength == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string error;
|
||||
|
||||
try
|
||||
{
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(fullPath, FileMode.Open);
|
||||
using var stream = mmf.CreateViewStream();
|
||||
using var br = new BinaryFileReader(stream);
|
||||
deserializer(br);
|
||||
using var accessor = mmf.CreateViewStream();
|
||||
|
||||
error = br.Position != fileLength
|
||||
? $"Serialized {fileLength} bytes, but {br.Position} bytes deserialized"
|
||||
byte* ptr = null;
|
||||
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
UnmanagedDataReader dataReader = new UnmanagedDataReader(ptr, accessor.Length);
|
||||
deserializer(dataReader);
|
||||
|
||||
error = dataReader.Position != fileLength
|
||||
? $"Serialized {fileLength} bytes, but {dataReader.Position} bytes deserialized"
|
||||
: null;
|
||||
|
||||
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,173 +0,0 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: BinaryFileReader.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* You should have received a copy of the GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using Server.Buffers;
|
||||
using Server.Logging;
|
||||
using Server.Text;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public class BinaryFileReader : IGenericReader, IDisposable
|
||||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(BinaryFileReader));
|
||||
|
||||
private Dictionary<ulong, string> _typesDb;
|
||||
private BinaryReader _reader;
|
||||
private Encoding _encoding;
|
||||
|
||||
public BinaryFileReader(BinaryReader br, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
|
||||
{
|
||||
_reader = br;
|
||||
_encoding = encoding ?? TextEncoding.UTF8;
|
||||
_typesDb = typesDb;
|
||||
}
|
||||
|
||||
public BinaryFileReader(Stream stream, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
|
||||
: this(new BinaryReader(stream), typesDb, encoding)
|
||||
{
|
||||
}
|
||||
|
||||
public long Position => _reader.BaseStream.Position;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Close() => _reader.Close();
|
||||
|
||||
public DateTime LastSerialized { get; init; }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null;
|
||||
|
||||
public string ReadStringRaw(bool intern = false)
|
||||
{
|
||||
var length = ((IGenericReader)this).ReadEncodedInt();
|
||||
if (length <= 0)
|
||||
{
|
||||
return "".Intern();
|
||||
}
|
||||
|
||||
byte[] buffer = STArrayPool<byte>.Shared.Rent(length);
|
||||
var strBuffer = buffer.AsSpan(0, length);
|
||||
_reader.Read(strBuffer);
|
||||
var str = TextEncoding.GetString(strBuffer, _encoding);
|
||||
STArrayPool<byte>.Shared.Return(buffer);
|
||||
return intern ? str.Intern() : str;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public long ReadLong() => _reader.ReadInt64();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ulong ReadULong() => _reader.ReadUInt64();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int ReadInt() => _reader.ReadInt32();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public uint ReadUInt() => _reader.ReadUInt32();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public short ReadShort() => _reader.ReadInt16();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ushort ReadUShort() => _reader.ReadUInt16();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public double ReadDouble() => _reader.ReadDouble();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public float ReadFloat() => _reader.ReadSingle();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte ReadByte() => _reader.ReadByte();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public sbyte ReadSByte() => _reader.ReadSByte();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool ReadBool() => _reader.ReadBoolean();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Serial ReadSerial() => (Serial)_reader.ReadUInt32();
|
||||
|
||||
public Type ReadType() =>
|
||||
ReadByte() switch
|
||||
{
|
||||
0 => null,
|
||||
1 => AssemblyHandler.FindTypeByFullName(ReadStringRaw()), // Backward compatibility
|
||||
2 => ReadTypeByHash()
|
||||
};
|
||||
|
||||
public Type ReadTypeByHash()
|
||||
{
|
||||
var hash = ReadULong();
|
||||
var t = AssemblyHandler.FindTypeByHash(hash);
|
||||
|
||||
if (t != null)
|
||||
{
|
||||
return t;
|
||||
}
|
||||
|
||||
if (_typesDb == null)
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"The file SerializedTypes.db was not loaded. Type hash '{hash}' could not be found."),
|
||||
"Invalid {Hash} at position {Position}",
|
||||
hash,
|
||||
Position
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!_typesDb.TryGetValue(hash, out var typeName))
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"Type hash '{hash}' is not present in the serialized types database."),
|
||||
"Invalid type hash {Hash} at position {Position}",
|
||||
hash,
|
||||
Position
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
t = AssemblyHandler.FindTypeByFullName(typeName, false);
|
||||
|
||||
if (t == null)
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"Type '{typeName}' was not found."),
|
||||
"Type {Type} was not found.",
|
||||
typeName
|
||||
);
|
||||
}
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int Read(Span<byte> buffer) => _reader.Read(buffer);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public long Seek(long offset, SeekOrigin origin) => _reader.BaseStream.Seek(offset, origin);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Dispose() => Close();
|
||||
}
|
||||
|
|
@ -1,101 +0,0 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: BinaryFileWriter.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* You should have received a copy of the GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public class BinaryFileWriter : BufferWriter, IDisposable
|
||||
{
|
||||
private readonly Stream _file;
|
||||
private long _position;
|
||||
|
||||
public BinaryFileWriter(string filename, bool prefixStr, ConcurrentQueue<Type> types = null) :
|
||||
this(new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None), prefixStr, types)
|
||||
{}
|
||||
|
||||
public BinaryFileWriter(Stream stream, bool prefixStr, ConcurrentQueue<Type> types = null) : base(prefixStr, types)
|
||||
{
|
||||
_file = stream;
|
||||
_position = _file.Position;
|
||||
}
|
||||
|
||||
public override long Position => _position + Index;
|
||||
|
||||
protected override int BufferSize => 81920;
|
||||
|
||||
public override void Flush()
|
||||
{
|
||||
if (Index > 0)
|
||||
{
|
||||
_position += Index;
|
||||
|
||||
_file.Write(Buffer, 0, (int)Index);
|
||||
Index = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
base.Flush(); // Increase buffer size
|
||||
}
|
||||
}
|
||||
|
||||
public override void Write(byte[] bytes) => Write(bytes, 0, bytes.Length);
|
||||
|
||||
public override void Write(byte[] bytes, int offset, int count)
|
||||
{
|
||||
if (Index > 0)
|
||||
{
|
||||
Flush();
|
||||
}
|
||||
|
||||
_file.Write(bytes, offset, count);
|
||||
_position += count;
|
||||
}
|
||||
|
||||
public override void Write(ReadOnlySpan<byte> bytes)
|
||||
{
|
||||
if (Index > 0)
|
||||
{
|
||||
Flush();
|
||||
}
|
||||
|
||||
_file.Write(bytes);
|
||||
_position += bytes.Length;
|
||||
}
|
||||
|
||||
public override void Close()
|
||||
{
|
||||
if (Index > 0)
|
||||
{
|
||||
Flush();
|
||||
}
|
||||
|
||||
_file.Close();
|
||||
}
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin)
|
||||
{
|
||||
if (Index > 0)
|
||||
{
|
||||
Flush();
|
||||
}
|
||||
|
||||
return _position = _file.Seek(offset, origin);
|
||||
}
|
||||
|
||||
public void Dispose() => Close();
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: BufferReader.cs *
|
||||
* *
|
||||
|
|
@ -29,9 +29,9 @@ public class BufferReader : IGenericReader
|
|||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(BufferReader));
|
||||
|
||||
private Dictionary<ulong, string> _typesDb;
|
||||
private Encoding _encoding;
|
||||
private byte[] _buffer;
|
||||
private readonly Dictionary<ulong, string> _typesDb;
|
||||
private readonly Encoding _encoding;
|
||||
private readonly byte[] _buffer;
|
||||
private int _position;
|
||||
|
||||
public long Position => _position;
|
||||
|
|
@ -44,21 +44,6 @@ public class BufferReader : IGenericReader
|
|||
_typesDb = typesDb;
|
||||
}
|
||||
|
||||
public BufferReader(byte[] buffer, DateTime lastSerialized, Dictionary<ulong, string> typesDb = null) : this(buffer)
|
||||
{
|
||||
LastSerialized = lastSerialized;
|
||||
_typesDb = typesDb;
|
||||
}
|
||||
|
||||
public void Reset(byte[] newBuffer, out byte[] oldBuffer)
|
||||
{
|
||||
oldBuffer = _buffer;
|
||||
_buffer = newBuffer;
|
||||
_position = 0;
|
||||
}
|
||||
|
||||
public DateTime LastSerialized { get; init; }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: BufferWriter.cs *
|
||||
* *
|
||||
|
|
@ -27,9 +27,9 @@ namespace Server;
|
|||
|
||||
public class BufferWriter : IGenericWriter
|
||||
{
|
||||
private ConcurrentQueue<Type> _types;
|
||||
private Encoding _encoding;
|
||||
private bool _prefixStrings;
|
||||
private readonly ConcurrentQueue<Type> _types;
|
||||
private readonly Encoding _encoding;
|
||||
private readonly bool _prefixStrings;
|
||||
private long _bytesWritten;
|
||||
private long _index;
|
||||
|
||||
|
|
|
|||
|
|
@ -14,10 +14,13 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using Server.Logging;
|
||||
|
|
@ -32,17 +35,23 @@ public interface IGenericEntityPersistence
|
|||
public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistence where T : class, ISerializable
|
||||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(GenericEntityPersistence<T>));
|
||||
private static List<EntitySpan<T>>[] _entities;
|
||||
|
||||
private static List<EntitySpan<T>> _entities;
|
||||
|
||||
private string _name;
|
||||
private long _initialIdxSize = 1024 * 256;
|
||||
private long _initialBinSize = 1024 * 1024;
|
||||
private readonly string _name;
|
||||
private readonly uint _minSerial;
|
||||
private readonly uint _maxSerial;
|
||||
private Serial _lastEntitySerial;
|
||||
private readonly Dictionary<Serial, T> _pendingAdd = new();
|
||||
private readonly Dictionary<Serial, T> _pendingDelete = new();
|
||||
private uint _minSerial;
|
||||
private uint _maxSerial;
|
||||
|
||||
public Dictionary<Serial, T> EntitiesBySerial { get; private set; } = new();
|
||||
private readonly uint[] _entitiesCount = new uint[World.GetThreadWorkerCount()];
|
||||
|
||||
private readonly (MemoryMapFileWriter idxWriter, MemoryMapFileWriter binWriter)[] _writers =
|
||||
new (MemoryMapFileWriter, MemoryMapFileWriter)[World.GetThreadWorkerCount()];
|
||||
|
||||
public Dictionary<Serial, T> EntitiesBySerial { get; } = new();
|
||||
|
||||
public GenericEntityPersistence(string name, int priority, uint minSerial, uint maxSerial) : base(priority)
|
||||
{
|
||||
|
|
@ -53,40 +62,414 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
typeof(T).RegisterFindEntity(Find);
|
||||
}
|
||||
|
||||
public override void Serialize()
|
||||
public override void Preserialize(string savePath, ConcurrentQueue<Type> types)
|
||||
{
|
||||
foreach (var entity in EntitiesBySerial.Values)
|
||||
var path = Path.Combine(savePath, _name);
|
||||
PathUtility.EnsureDirectory(path);
|
||||
|
||||
var threadCount = World.GetThreadWorkerCount();
|
||||
for (var i = 0; i < threadCount; i++)
|
||||
{
|
||||
World.PushToCache(entity);
|
||||
var idxPath = Path.Combine(path, $"{_name}_{i}.idx");
|
||||
var binPath = Path.Combine(path, $"{_name}_{i}.bin");
|
||||
|
||||
_writers[i] = (
|
||||
new MemoryMapFileWriter(new FileStream(idxPath, FileMode.Create), _initialIdxSize, types),
|
||||
new MemoryMapFileWriter(new FileStream(binPath, FileMode.Create), _initialBinSize, types)
|
||||
);
|
||||
|
||||
_writers[i].idxWriter.Write(3); // version
|
||||
_writers[i].idxWriter.Seek(4, SeekOrigin.Current); // Entity count
|
||||
|
||||
_entitiesCount[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public override void WriteSnapshot(string basePath)
|
||||
public override void Serialize(IGenericSerializable e, int threadIndex)
|
||||
{
|
||||
IIndexInfo<Serial> indexInfo = new EntityTypeIndex(_name);
|
||||
EntityPersistence.WriteEntities(indexInfo, EntitiesBySerial, basePath,World.SerializedTypes);
|
||||
var (idx, bin) = _writers[threadIndex];
|
||||
var pos = bin.Position;
|
||||
|
||||
var entity = (ISerializable)e;
|
||||
|
||||
entity.Serialize(bin);
|
||||
var length = (uint)(bin.Position - pos);
|
||||
|
||||
var t = entity.GetType();
|
||||
idx.Write(t);
|
||||
idx.Write(entity.Serial);
|
||||
idx.Write(entity.Created.Ticks);
|
||||
idx.Write(pos);
|
||||
idx.Write(length);
|
||||
|
||||
_entitiesCount[threadIndex]++;
|
||||
}
|
||||
|
||||
public override void WriteSnapshot()
|
||||
{
|
||||
var wroteFile = false;
|
||||
string folderPath = null;
|
||||
for (int i = 0; i < _writers.Length; i++)
|
||||
{
|
||||
var (idxWriter, binWriter) = _writers[i];
|
||||
|
||||
var binBytesWritten = binWriter.Position;
|
||||
|
||||
// Write the entity count
|
||||
var pos = idxWriter.Position;
|
||||
idxWriter.Seek(4, SeekOrigin.Begin);
|
||||
idxWriter.Write(_entitiesCount[i]);
|
||||
idxWriter.Seek(pos, SeekOrigin.Begin);
|
||||
|
||||
var idxFs = idxWriter.FileStream;
|
||||
var idxFilePath = idxFs.Name;
|
||||
var binFs = binWriter.FileStream;
|
||||
var binFilePath = binFs.Name;
|
||||
|
||||
if (_initialIdxSize < idxFs.Position)
|
||||
{
|
||||
_initialIdxSize = idxFs.Position;
|
||||
}
|
||||
|
||||
if (_initialBinSize < binFs.Position)
|
||||
{
|
||||
_initialBinSize = binFs.Position;
|
||||
}
|
||||
|
||||
idxWriter.Dispose();
|
||||
binWriter.Dispose();
|
||||
|
||||
idxFs.Dispose();
|
||||
binFs.Dispose();
|
||||
|
||||
if (binBytesWritten > 1)
|
||||
{
|
||||
wroteFile = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
File.Delete(idxFilePath);
|
||||
File.Delete(binFilePath);
|
||||
folderPath = Path.GetDirectoryName(idxFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
if (!wroteFile && folderPath != null)
|
||||
{
|
||||
Directory.Delete(folderPath);
|
||||
}
|
||||
}
|
||||
|
||||
public override void Serialize()
|
||||
{
|
||||
World.ResetRoundRobin();
|
||||
foreach (var entity in EntitiesBySerial.Values)
|
||||
{
|
||||
World.PushToCache((entity, this));
|
||||
}
|
||||
}
|
||||
|
||||
private static ConstructorInfo GetConstructorFor(string typeName, Type t, Type[] constructorTypes)
|
||||
{
|
||||
if (t?.IsAbstract != false)
|
||||
{
|
||||
Console.WriteLine("failed");
|
||||
|
||||
var issue = t?.IsAbstract == true ? "marked abstract" : "not found";
|
||||
|
||||
Console.Write($"Error: Type '{typeName}' was {issue}. Delete all of those types? (y/n): ");
|
||||
|
||||
if (Console.ReadLine().InsensitiveEquals("y"))
|
||||
{
|
||||
Console.WriteLine("Loading...");
|
||||
return null;
|
||||
}
|
||||
|
||||
Console.WriteLine("Types will not be deleted. An exception will be thrown.");
|
||||
|
||||
throw new Exception($"Bad type '{typeName}'");
|
||||
}
|
||||
|
||||
var ctor = t.GetConstructor(constructorTypes);
|
||||
|
||||
if (ctor == null)
|
||||
{
|
||||
throw new Exception($"Type '{t}' does not have a serialization constructor");
|
||||
}
|
||||
|
||||
return ctor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy ReadTypes for backward compatibility with old saves that still have a tdb file
|
||||
*/
|
||||
private unsafe Dictionary<int, ConstructorInfo> ReadTypes(string savePath)
|
||||
{
|
||||
string typesPath = Path.Combine(savePath, _name, $"{_name}.tdb");
|
||||
if (!File.Exists(typesPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
Type[] ctorArguments = [typeof(Serial)];
|
||||
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(typesPath, FileMode.Open);
|
||||
using var accessor = mmf.CreateViewStream();
|
||||
|
||||
byte* ptr = null;
|
||||
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
var dataReader = new UnmanagedDataReader(ptr, accessor.Length);
|
||||
|
||||
var count = dataReader.ReadInt();
|
||||
var types = new Dictionary<int, ConstructorInfo>(count);
|
||||
|
||||
for (var i = 0; i < count; ++i)
|
||||
{
|
||||
// Legacy didn't have the null flag check
|
||||
var typeName = dataReader.ReadStringRaw();
|
||||
types.Add(i, GetConstructorFor(typeName, AssemblyHandler.FindTypeByName(typeName), ctorArguments));
|
||||
}
|
||||
|
||||
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
return types;
|
||||
}
|
||||
|
||||
public virtual void DeserializeIndexes(string savePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
IIndexInfo<Serial> indexInfo = new EntityTypeIndex(_name);
|
||||
string indexPath = Path.Combine(savePath, _name, $"{_name}.idx");
|
||||
if (!File.Exists(indexPath))
|
||||
{
|
||||
TryDeserializeMultithreadIndexes(savePath, typesDb);
|
||||
return;
|
||||
}
|
||||
|
||||
EntitiesBySerial = EntityPersistence.LoadIndex(savePath, indexInfo, typesDb, out _entities);
|
||||
_entities = [InternalDeserializeIndexes(indexPath, typesDb)];
|
||||
}
|
||||
|
||||
private void TryDeserializeMultithreadIndexes(string savePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
var index = 0;
|
||||
var fileList = new List<string>();
|
||||
while (true)
|
||||
{
|
||||
var path = Path.Combine(savePath, _name, $"{_name}_{index}.idx");
|
||||
var fi = new FileInfo(path);
|
||||
if (!fi.Exists)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (fi.Length != 0)
|
||||
{
|
||||
fileList.Add(path);
|
||||
}
|
||||
|
||||
index++;
|
||||
}
|
||||
|
||||
_entities = new List<EntitySpan<T>>[fileList.Count];
|
||||
for (var i = 0; i < fileList.Count; i++)
|
||||
{
|
||||
_entities[i] = InternalDeserializeIndexes(fileList[i], typesDb);
|
||||
}
|
||||
}
|
||||
|
||||
private unsafe List<EntitySpan<T>> InternalDeserializeIndexes(string filePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
object[] ctorArgs = new object[1];
|
||||
List<EntitySpan<T>> entities = [];
|
||||
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(filePath, FileMode.Open);
|
||||
using var accessor = mmf.CreateViewStream();
|
||||
|
||||
byte* ptr = null;
|
||||
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
UnmanagedDataReader dataReader = new UnmanagedDataReader(ptr, accessor.Length, typesDb);
|
||||
|
||||
var version = dataReader.ReadInt();
|
||||
|
||||
Dictionary<int, ConstructorInfo> ctors = null;
|
||||
if (version < 2)
|
||||
{
|
||||
ctors = ReadTypes(Path.GetDirectoryName(filePath));
|
||||
}
|
||||
|
||||
if (typesDb == null && ctors == null)
|
||||
{
|
||||
return entities;
|
||||
}
|
||||
|
||||
int count = dataReader.ReadInt();
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
Type[] ctorArguments = [typeof(Serial)];
|
||||
|
||||
for (int i = 0; i < count; ++i)
|
||||
{
|
||||
ConstructorInfo ctor;
|
||||
// Version 2 & 3 with SerializedTypes.db
|
||||
if (version >= 2)
|
||||
{
|
||||
var flag = dataReader.ReadByte();
|
||||
if (flag != 2)
|
||||
{
|
||||
throw new Exception($"Invalid type flag, expected 2 but received {flag}.");
|
||||
}
|
||||
|
||||
var hash = dataReader.ReadULong();
|
||||
typesDb!.TryGetValue(hash, out var typeName);
|
||||
ctor = GetConstructorFor(typeName, AssemblyHandler.FindTypeByHash(hash), ctorArguments);
|
||||
}
|
||||
else
|
||||
{
|
||||
ctor = ctors?[dataReader.ReadInt()];
|
||||
}
|
||||
|
||||
Serial serial = (Serial)dataReader.ReadUInt();
|
||||
var created = version == 0 ? now : new DateTime(dataReader.ReadLong(), DateTimeKind.Utc);
|
||||
if (version is > 0 and < 3)
|
||||
{
|
||||
dataReader.ReadLong(); // LastSerialized
|
||||
}
|
||||
var pos = dataReader.ReadLong();
|
||||
var length = dataReader.ReadInt();
|
||||
|
||||
if (ctor == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
ctorArgs[0] = serial;
|
||||
|
||||
if (ctor.Invoke(ctorArgs) is T entity)
|
||||
{
|
||||
entity.Created = created;
|
||||
entities.Add(new EntitySpan<T>(entity, pos, (int)length));
|
||||
EntitiesBySerial[serial] = entity;
|
||||
}
|
||||
}
|
||||
|
||||
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
entities.TrimExcess();
|
||||
|
||||
if (EntitiesBySerial.Count > 0)
|
||||
{
|
||||
_lastEntitySerial = EntitiesBySerial.Keys.Max();
|
||||
}
|
||||
|
||||
return entities;
|
||||
}
|
||||
|
||||
public override void Deserialize(string savePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
IIndexInfo<Serial> indexInfo = new EntityTypeIndex(_name);
|
||||
EntityPersistence.LoadData(savePath, indexInfo, typesDb, _entities);
|
||||
string dataPath = Path.Combine(savePath, _name, $"{_name}.bin");
|
||||
var fi = new FileInfo(dataPath);
|
||||
|
||||
if (!fi.Exists)
|
||||
{
|
||||
TryDeserializeMultithread(savePath, typesDb);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (fi.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
InternalDeserialize(dataPath, 0, typesDb);
|
||||
}
|
||||
|
||||
_entities = null;
|
||||
}
|
||||
|
||||
public override void PostSerialize()
|
||||
private static unsafe void InternalDeserialize(string filePath, int index, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(filePath, FileMode.Open);
|
||||
using var accessor = mmf.CreateViewStream();
|
||||
|
||||
byte* ptr = null;
|
||||
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
UnmanagedDataReader dataReader = new UnmanagedDataReader(ptr, accessor.Length, typesDb);
|
||||
var deleteAllFailures = false;
|
||||
|
||||
foreach (var entry in _entities[index])
|
||||
{
|
||||
T t = entry.Entity;
|
||||
|
||||
if (entry.Length == 0)
|
||||
{
|
||||
t?.Delete();
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip this entry
|
||||
if (t == null)
|
||||
{
|
||||
dataReader.Seek(entry.Length, SeekOrigin.Current);
|
||||
continue;
|
||||
}
|
||||
|
||||
string error;
|
||||
|
||||
try
|
||||
{
|
||||
var pos = dataReader.Position;
|
||||
t.Deserialize(dataReader);
|
||||
var lengthDeserialized = dataReader.Position - pos;
|
||||
|
||||
error = lengthDeserialized != entry.Length
|
||||
? $"Serialized object was {entry.Length} bytes, but {lengthDeserialized} bytes deserialized"
|
||||
: null;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
error = e.ToString();
|
||||
}
|
||||
|
||||
if (error != null)
|
||||
{
|
||||
Console.WriteLine($"***** Bad deserialize of {t.GetType()} ({t.Serial}) *****");
|
||||
Console.WriteLine(error);
|
||||
|
||||
if (!deleteAllFailures)
|
||||
{
|
||||
Console.Write("Delete the object and continue? (y/n/a): ");
|
||||
var pressedKey = Console.ReadLine();
|
||||
|
||||
if (pressedKey.InsensitiveEquals("a"))
|
||||
{
|
||||
deleteAllFailures = true;
|
||||
}
|
||||
else if (!pressedKey.InsensitiveEquals("y"))
|
||||
{
|
||||
throw new Exception("Deserialization failed.");
|
||||
}
|
||||
}
|
||||
|
||||
t.Delete();
|
||||
}
|
||||
}
|
||||
|
||||
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
}
|
||||
|
||||
private void TryDeserializeMultithread(string savePath, Dictionary<ulong, string> typesDb)
|
||||
{
|
||||
if (_entities == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var folderPath = Path.Combine(savePath, _name);
|
||||
|
||||
for (var i = 0; i < _entities.Length; i++)
|
||||
{
|
||||
var path = Path.Combine(folderPath, $"{_name}_{i}.bin");
|
||||
InternalDeserialize(path, i, typesDb);
|
||||
}
|
||||
}
|
||||
|
||||
public override void PostWorldSave()
|
||||
{
|
||||
ProcessSafetyQueues();
|
||||
}
|
||||
|
|
@ -144,10 +527,9 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
case WorldState.Saving:
|
||||
{
|
||||
AppendSafetyLog("add", entity);
|
||||
goto case WorldState.WritingSave;
|
||||
goto case WorldState.Loading;
|
||||
}
|
||||
case WorldState.Loading:
|
||||
case WorldState.WritingSave:
|
||||
{
|
||||
if (_pendingDelete.Remove(entity.Serial))
|
||||
{
|
||||
|
|
@ -158,6 +540,7 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
break;
|
||||
}
|
||||
case WorldState.PendingSave:
|
||||
case WorldState.WritingSave:
|
||||
case WorldState.Running:
|
||||
{
|
||||
ref var entityEntry = ref CollectionsMarshal.GetValueRefOrAddDefault(EntitiesBySerial, entity.Serial, out bool exists);
|
||||
|
|
@ -205,16 +588,16 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
case WorldState.Saving:
|
||||
{
|
||||
AppendSafetyLog("delete", entity);
|
||||
goto case WorldState.WritingSave;
|
||||
goto case WorldState.Loading;
|
||||
}
|
||||
case WorldState.Loading:
|
||||
case WorldState.WritingSave:
|
||||
{
|
||||
_pendingAdd.Remove(entity.Serial);
|
||||
_pendingDelete[entity.Serial] = entity;
|
||||
break;
|
||||
}
|
||||
case WorldState.PendingSave:
|
||||
case WorldState.WritingSave:
|
||||
case WorldState.Running:
|
||||
{
|
||||
EntitiesBySerial.Remove(entity.Serial);
|
||||
|
|
@ -286,7 +669,6 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
}
|
||||
case WorldState.Loading:
|
||||
case WorldState.Saving:
|
||||
case WorldState.WritingSave:
|
||||
{
|
||||
if (returnDeleted && returnPending && _pendingDelete.TryGetValue(serial, out var entity))
|
||||
{
|
||||
|
|
@ -302,6 +684,7 @@ public class GenericEntityPersistence<T> : Persistence, IGenericEntityPersistenc
|
|||
return null;
|
||||
}
|
||||
case WorldState.PendingSave:
|
||||
case WorldState.WritingSave:
|
||||
case WorldState.Running:
|
||||
{
|
||||
return EntitiesBySerial.TryGetValue(serial, out var entity) ? entity as R : null;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: GenericPersistence.cs *
|
||||
* *
|
||||
|
|
@ -22,36 +22,55 @@ namespace Server;
|
|||
|
||||
public abstract class GenericPersistence : Persistence, IGenericSerializable
|
||||
{
|
||||
private long _initialSize = 1024 * 1024;
|
||||
private MemoryMapFileWriter _fileToSave;
|
||||
|
||||
public string Name { get; }
|
||||
|
||||
public GenericPersistence(string name, int priority) : base(priority) => Name = name;
|
||||
|
||||
public override void Preserialize(string savePath, ConcurrentQueue<Type> types)
|
||||
{
|
||||
var path = Path.Combine(savePath, Name);
|
||||
var filePath = Path.Combine(path, $"{Name}.bin");
|
||||
PathUtility.EnsureDirectory(path);
|
||||
|
||||
_fileToSave = new MemoryMapFileWriter(new FileStream(filePath, FileMode.Create), _initialSize, types);
|
||||
}
|
||||
|
||||
public override void Serialize()
|
||||
{
|
||||
World.PushToCache(this);
|
||||
World.ResetRoundRobin();
|
||||
World.PushToCache((this, this));
|
||||
}
|
||||
|
||||
public long SavePosition { get; set; }
|
||||
|
||||
public BufferWriter SaveBuffer { get; set; }
|
||||
|
||||
public void Serialize(ConcurrentQueue<Type> types)
|
||||
public override void WriteSnapshot()
|
||||
{
|
||||
SaveBuffer ??= new BufferWriter(true, types);
|
||||
string folderPath = null;
|
||||
using (var fs = _fileToSave.FileStream)
|
||||
{
|
||||
if (fs.Position > _initialSize)
|
||||
{
|
||||
_initialSize = fs.Position;
|
||||
}
|
||||
|
||||
SaveBuffer.Seek(0, SeekOrigin.Begin);
|
||||
Serialize(SaveBuffer);
|
||||
_fileToSave.Dispose();
|
||||
if (_fileToSave.Position == 0)
|
||||
{
|
||||
folderPath = Path.GetDirectoryName(fs.Name);
|
||||
}
|
||||
}
|
||||
|
||||
if (folderPath != null)
|
||||
{
|
||||
Directory.Delete(folderPath);
|
||||
}
|
||||
}
|
||||
|
||||
public override void Serialize(IGenericSerializable e, int threadIndex) => Serialize(_fileToSave);
|
||||
|
||||
public abstract void Serialize(IGenericWriter writer);
|
||||
|
||||
public override void WriteSnapshot(string basePath)
|
||||
{
|
||||
string binPath = Path.Combine(basePath, Name, $"{Name}.bin");
|
||||
var buffer = SaveBuffer!.Buffer.AsSpan(0, (int)SaveBuffer.Position);
|
||||
AdhocPersistence.WriteSnapshot(new FileInfo(binPath), buffer);
|
||||
}
|
||||
|
||||
public override void Deserialize(string savePath, Dictionary<ulong, string> typesDb) =>
|
||||
AdhocPersistence.Deserialize(Path.Combine(savePath, Name, $"{Name}.bin"), Deserialize);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: IGenericReader.cs *
|
||||
* *
|
||||
|
|
@ -22,9 +22,6 @@ namespace Server;
|
|||
|
||||
public interface IGenericReader
|
||||
{
|
||||
// Used to determine valid Entity deserialization
|
||||
DateTime LastSerialized { get; init; }
|
||||
|
||||
string ReadString(bool intern = false);
|
||||
public string ReadStringRaw(bool intern = false);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: IGenericWriter.cs *
|
||||
* *
|
||||
|
|
@ -24,7 +24,6 @@ namespace Server;
|
|||
public interface IGenericWriter
|
||||
{
|
||||
long Position { get; }
|
||||
void Close();
|
||||
void Write(string value);
|
||||
void Write(long value);
|
||||
void Write(ulong value);
|
||||
|
|
@ -98,6 +97,7 @@ public interface IGenericWriter
|
|||
|
||||
Write((byte)v);
|
||||
}
|
||||
|
||||
void Write(Point3D value)
|
||||
{
|
||||
Write(value.m_X);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: ISerializable.cs *
|
||||
* *
|
||||
|
|
@ -14,61 +14,18 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public interface ISerializable : IGenericSerializable
|
||||
{
|
||||
// Should be serialized/deserialized with the index so it can be referenced by IGenericReader
|
||||
// Should be serialized/deserialized with the index that way it can be referenced by IGenericReader
|
||||
DateTime Created { get; set; }
|
||||
|
||||
long SavePosition { get; protected internal set; }
|
||||
BufferWriter SaveBuffer { get; protected internal set; }
|
||||
|
||||
Serial Serial { get; }
|
||||
|
||||
void Deserialize(IGenericReader reader);
|
||||
void Serialize(IGenericWriter writer);
|
||||
|
||||
bool Deleted { get; }
|
||||
void Delete();
|
||||
|
||||
public void InitializeSaveBuffer(byte[] buffer, ConcurrentQueue<Type> types)
|
||||
{
|
||||
SaveBuffer = new BufferWriter(buffer, true, types);
|
||||
if (World.DirtyTrackingEnabled)
|
||||
{
|
||||
SavePosition = SaveBuffer.Position;
|
||||
}
|
||||
else
|
||||
{
|
||||
SavePosition = -1;
|
||||
}
|
||||
}
|
||||
|
||||
void IGenericSerializable.Serialize(ConcurrentQueue<Type> types)
|
||||
{
|
||||
SaveBuffer ??= new BufferWriter(true, types);
|
||||
|
||||
// Clean, don't bother serializing
|
||||
if (SavePosition > -1)
|
||||
{
|
||||
SaveBuffer.Seek(SavePosition, SeekOrigin.Begin);
|
||||
return;
|
||||
}
|
||||
|
||||
SaveBuffer.Seek(0, SeekOrigin.Begin);
|
||||
Serialize(SaveBuffer);
|
||||
|
||||
if (World.DirtyTrackingEnabled)
|
||||
{
|
||||
SavePosition = SaveBuffer.Position;
|
||||
}
|
||||
else
|
||||
{
|
||||
this.MarkDirty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: ISerializableExtensions.cs *
|
||||
* *
|
||||
|
|
@ -24,10 +24,7 @@ public static class ISerializableExtensions
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void MarkDirty(this ISerializable entity)
|
||||
{
|
||||
if (entity != null)
|
||||
{
|
||||
entity.SavePosition = -1;
|
||||
}
|
||||
// TODO: Add dirty tracking back
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
|
|
|
|||
304
Projects/Server/Serialization/MemoryMapFileWriter.cs
Normal file
304
Projects/Server/Serialization/MemoryMapFileWriter.cs
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: MemoryMapFileWriter.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* You should have received a copy of the GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using Server.Text;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public unsafe class MemoryMapFileWriter : IGenericWriter, IDisposable
|
||||
{
|
||||
private readonly Encoding _encoding;
|
||||
|
||||
private readonly ConcurrentQueue<Type> _types;
|
||||
private readonly FileStream _fileStream;
|
||||
private MemoryMappedFile _mmf;
|
||||
private MemoryMappedViewAccessor _accessor;
|
||||
private byte* _ptr;
|
||||
private long _position;
|
||||
private long _size;
|
||||
|
||||
public MemoryMapFileWriter(FileStream fileStream, long initialSize, ConcurrentQueue<Type> types = null)
|
||||
{
|
||||
_types = types;
|
||||
_fileStream = fileStream;
|
||||
_encoding = TextEncoding.UTF8;
|
||||
_size = Math.Max(initialSize, 1024);
|
||||
|
||||
ResizeMemoryMappedFile(initialSize);
|
||||
}
|
||||
|
||||
public long Position => _position;
|
||||
|
||||
public FileStream FileStream => _fileStream;
|
||||
|
||||
private void ResizeMemoryMappedFile(long newSize)
|
||||
{
|
||||
_accessor?.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
_accessor?.Dispose();
|
||||
_mmf?.Dispose();
|
||||
|
||||
// Do the actual resizing
|
||||
_fileStream.SetLength(newSize);
|
||||
|
||||
_mmf = MemoryMappedFile.CreateFromFile(_fileStream, null, newSize, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, leaveOpen: true);
|
||||
_accessor = _mmf.CreateViewAccessor();
|
||||
_accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref _ptr);
|
||||
}
|
||||
|
||||
private void EnsureCapacity(long bytesToWrite)
|
||||
{
|
||||
var shouldResize = false;
|
||||
while (_position + bytesToWrite > _size)
|
||||
{
|
||||
// Don't double forever, eventually we want to have a maximum, like 256MB at a time or something
|
||||
_size += Math.Min(_size, 1024 * 1024 * 256);
|
||||
shouldResize = true;
|
||||
}
|
||||
|
||||
if (shouldResize)
|
||||
{
|
||||
ResizeMemoryMappedFile(_size);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(byte[] bytes) => Write(bytes.AsSpan());
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(byte[] bytes, int offset, int count) => Write(bytes.AsSpan(offset, count));
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(ReadOnlySpan<byte> bytes)
|
||||
{
|
||||
var byteCount = bytes.Length;
|
||||
EnsureCapacity(byteCount);
|
||||
|
||||
bytes.CopyTo(new Span<byte>(_ptr + _position, byteCount));
|
||||
_position += byteCount;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public virtual long Seek(long offset, SeekOrigin origin)
|
||||
{
|
||||
switch (origin)
|
||||
{
|
||||
case SeekOrigin.Begin:
|
||||
{
|
||||
if (offset > _size)
|
||||
{
|
||||
EnsureCapacity(offset);
|
||||
}
|
||||
|
||||
_position = offset;
|
||||
break;
|
||||
}
|
||||
case SeekOrigin.Current:
|
||||
{
|
||||
EnsureCapacity(offset);
|
||||
_position += offset;
|
||||
break;
|
||||
}
|
||||
case SeekOrigin.End:
|
||||
{
|
||||
if (_position + offset > _size)
|
||||
{
|
||||
EnsureCapacity(offset);
|
||||
}
|
||||
|
||||
_position = _size + offset;
|
||||
|
||||
if (_position < 0)
|
||||
{
|
||||
Dispose();
|
||||
throw new InvalidOperationException("Seek before start of file");
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return _position;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(string value)
|
||||
{
|
||||
if (value == null)
|
||||
{
|
||||
Write(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
Write(true);
|
||||
WriteStringRaw(value);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(long value)
|
||||
{
|
||||
EnsureCapacity(sizeof(long));
|
||||
BinaryPrimitives.WriteInt64LittleEndian(new Span<byte>(_ptr + _position, sizeof(long)), value);
|
||||
_position += sizeof(long);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(ulong value)
|
||||
{
|
||||
EnsureCapacity(sizeof(ulong));
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(new Span<byte>(_ptr + _position, sizeof(ulong)), value);
|
||||
_position += sizeof(ulong);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(int value)
|
||||
{
|
||||
EnsureCapacity(sizeof(int));
|
||||
BinaryPrimitives.WriteInt32LittleEndian(new Span<byte>(_ptr + _position, sizeof(int)), value);
|
||||
_position += sizeof(int);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(uint value)
|
||||
{
|
||||
EnsureCapacity(sizeof(uint));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(new Span<byte>(_ptr + _position, sizeof(uint)), value);
|
||||
_position += sizeof(uint);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(short value)
|
||||
{
|
||||
EnsureCapacity(sizeof(short));
|
||||
BinaryPrimitives.WriteInt16LittleEndian(new Span<byte>(_ptr + _position, sizeof(short)), value);
|
||||
_position += sizeof(short);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(ushort value)
|
||||
{
|
||||
EnsureCapacity(sizeof(ushort));
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(new Span<byte>(_ptr + _position, sizeof(ushort)), value);
|
||||
_position += sizeof(ushort);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(double value)
|
||||
{
|
||||
EnsureCapacity(sizeof(double));
|
||||
BinaryPrimitives.WriteDoubleLittleEndian(new Span<byte>(_ptr + _position, sizeof(double)), value);
|
||||
_position += sizeof(double);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(float value)
|
||||
{
|
||||
EnsureCapacity(sizeof(float));
|
||||
BinaryPrimitives.WriteSingleLittleEndian(new Span<byte>(_ptr + _position, sizeof(float)), value);
|
||||
_position += sizeof(float);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(byte value)
|
||||
{
|
||||
EnsureCapacity(1);
|
||||
*(_ptr + _position++) = value;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(sbyte value) => Write((byte)value);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(bool value) => Write(*(byte*)&value);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(Serial serial) => Write(serial.Value);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(Type type)
|
||||
{
|
||||
if (type == null)
|
||||
{
|
||||
Write((byte)0);
|
||||
}
|
||||
else
|
||||
{
|
||||
Write((byte)0x2); // xxHash3 64bit
|
||||
Write(AssemblyHandler.GetTypeHash(type));
|
||||
_types.Enqueue(type);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Write(decimal value)
|
||||
{
|
||||
Span<int> buffer = stackalloc int[sizeof(decimal) / 4];
|
||||
decimal.GetBits(value, buffer);
|
||||
|
||||
Write(MemoryMarshal.Cast<int, byte>(buffer));
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void WriteStringRaw(ReadOnlySpan<char> value)
|
||||
{
|
||||
var length = _encoding.GetByteCount(value);
|
||||
|
||||
EnsureCapacity(length + 5);
|
||||
|
||||
// WriteEncodedInt
|
||||
var v = (uint)length;
|
||||
|
||||
while (v >= 0x80)
|
||||
{
|
||||
*(_ptr + _position++) = (byte)(v | 0x80);
|
||||
v >>= 7;
|
||||
}
|
||||
*(_ptr + _position++) = (byte)v;
|
||||
|
||||
_encoding.GetBytes(value, new Span<byte>(_ptr + _position, length));
|
||||
_position += length;
|
||||
}
|
||||
|
||||
private void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
_accessor.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
_accessor.Dispose();
|
||||
_mmf.Dispose();
|
||||
|
||||
// Truncate the file
|
||||
_fileStream.SetLength(_position);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
~MemoryMapFileWriter()
|
||||
{
|
||||
Dispose(false);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2023 - ModernUO Development Team *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: Persistence.cs *
|
||||
* *
|
||||
|
|
@ -17,6 +17,7 @@ using System;
|
|||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
|
||||
namespace Server;
|
||||
|
||||
|
|
@ -52,61 +53,54 @@ public abstract class Persistence
|
|||
}
|
||||
}
|
||||
|
||||
private static Dictionary<ulong, string> LoadTypes(string path)
|
||||
private unsafe static Dictionary<ulong, string> LoadTypes(string path)
|
||||
{
|
||||
var db = new Dictionary<ulong, string>();
|
||||
|
||||
string tdbPath = Path.Combine(path, "SerializedTypes.db");
|
||||
if (!File.Exists(tdbPath))
|
||||
string typesPath = Path.Combine(path, "SerializedTypes.db");
|
||||
if (!File.Exists(typesPath))
|
||||
{
|
||||
return db;
|
||||
}
|
||||
|
||||
using FileStream tdb = new FileStream(tdbPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
BinaryReader tdbReader = new BinaryReader(tdb);
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(typesPath, FileMode.Open);
|
||||
using var accessor = mmf.CreateViewStream();
|
||||
|
||||
var version = tdbReader.ReadInt32();
|
||||
var count = tdbReader.ReadInt32();
|
||||
byte* ptr = null;
|
||||
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
|
||||
var dataReader = new UnmanagedDataReader(ptr, accessor.Length);
|
||||
|
||||
var version = dataReader.ReadInt();
|
||||
var count = dataReader.ReadInt();
|
||||
|
||||
for (var i = 0; i < count; ++i)
|
||||
{
|
||||
var hash = tdbReader.ReadUInt64();
|
||||
var typeName = tdbReader.ReadString();
|
||||
var hash = dataReader.ReadULong();
|
||||
var typeName = dataReader.ReadStringRaw();
|
||||
db[hash] = typeName;
|
||||
}
|
||||
|
||||
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
|
||||
return db;
|
||||
}
|
||||
|
||||
internal static void SerializeAll()
|
||||
// Note: This is strictly on a background thread
|
||||
internal static void PreSerializeAll(string path, ConcurrentQueue<Type> types)
|
||||
{
|
||||
foreach (var p in _registry)
|
||||
{
|
||||
p.Serialize();
|
||||
p.Preserialize(path, types);
|
||||
}
|
||||
}
|
||||
|
||||
internal static void PostSerializeAll()
|
||||
private static readonly HashSet<Type> _typesSet = [];
|
||||
|
||||
// Note: This is strictly on a background thread
|
||||
internal static void WriteSnapshotAll(string path, ConcurrentQueue<Type> types)
|
||||
{
|
||||
foreach (var p in _registry)
|
||||
{
|
||||
p.PostSerialize();
|
||||
}
|
||||
}
|
||||
|
||||
internal static void PostDeserializeAll()
|
||||
{
|
||||
foreach (var p in _registry)
|
||||
{
|
||||
p.PostDeserialize();
|
||||
}
|
||||
}
|
||||
|
||||
public static void WriteSnapshot(string path, ConcurrentQueue<Type> types)
|
||||
{
|
||||
foreach (var entry in _registry)
|
||||
{
|
||||
entry.WriteSnapshot(path);
|
||||
p.WriteSnapshot();
|
||||
}
|
||||
|
||||
// Dedupe the queue.
|
||||
|
|
@ -119,32 +113,62 @@ public abstract class Persistence
|
|||
_typesSet.Clear();
|
||||
}
|
||||
|
||||
private static HashSet<Type> _typesSet = new();
|
||||
internal static void SerializeAll()
|
||||
{
|
||||
foreach (var p in _registry)
|
||||
{
|
||||
p.Serialize();
|
||||
}
|
||||
}
|
||||
|
||||
internal static void PostWorldSaveAll()
|
||||
{
|
||||
foreach (var p in _registry)
|
||||
{
|
||||
p.PostWorldSave();
|
||||
}
|
||||
}
|
||||
|
||||
internal static void PostDeserializeAll()
|
||||
{
|
||||
foreach (var p in _registry)
|
||||
{
|
||||
p.PostDeserialize();
|
||||
}
|
||||
}
|
||||
|
||||
public static void WriteSerializedTypesSnapshot(string path, HashSet<Type> types)
|
||||
{
|
||||
string tdbPath = Path.Combine(path, "SerializedTypes.db");
|
||||
using var tdb = new BinaryFileWriter(tdbPath, false);
|
||||
string typesPath = Path.Combine(path, "SerializedTypes.db");
|
||||
using var fs = new FileStream(typesPath, FileMode.Create);
|
||||
using var writer = new MemoryMapFileWriter(fs, 1024 * 1024 * 4);
|
||||
|
||||
tdb.Write(0); // version
|
||||
tdb.Write(types.Count);
|
||||
writer.Write(0); // version
|
||||
writer.Write(types.Count);
|
||||
|
||||
foreach (var type in types)
|
||||
{
|
||||
var fullName = type.FullName;
|
||||
tdb.Write(HashUtility.ComputeHash64(fullName));
|
||||
tdb.Write(fullName);
|
||||
writer.Write(HashUtility.ComputeHash64(fullName));
|
||||
writer.WriteStringRaw(fullName);
|
||||
}
|
||||
}
|
||||
|
||||
// Serializes to memory buffers and run in parallel
|
||||
public abstract void Serialize();
|
||||
// Open file streams, MMFs, prepare data structures
|
||||
// Note: This should only be run on a background thread
|
||||
public abstract void Preserialize(string savePath, ConcurrentQueue<Type> types);
|
||||
|
||||
public abstract void WriteSnapshot(string savePath);
|
||||
// Note: This should only be run on a background thread
|
||||
public abstract void Serialize(IGenericSerializable e, int threadIndex);
|
||||
|
||||
// Note: This should only be run on a background thread
|
||||
public abstract void WriteSnapshot();
|
||||
|
||||
public abstract void Serialize();
|
||||
|
||||
public abstract void Deserialize(string savePath, Dictionary<ulong, string> typesDb);
|
||||
|
||||
public virtual void PostSerialize()
|
||||
public virtual void PostWorldSave()
|
||||
{
|
||||
}
|
||||
|
||||
|
|
|
|||
246
Projects/Server/Serialization/UnmanagedDataReader.cs
Normal file
246
Projects/Server/Serialization/UnmanagedDataReader.cs
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: UnmanagedDataReader.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* You should have received a copy of the GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using Server.Logging;
|
||||
using Server.Text;
|
||||
|
||||
namespace Server;
|
||||
|
||||
public unsafe class UnmanagedDataReader : IGenericReader
|
||||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(UnmanagedDataReader));
|
||||
|
||||
private readonly byte* _ptr;
|
||||
private long _position;
|
||||
private readonly long _size;
|
||||
|
||||
private readonly Dictionary<ulong, string> _typesDb;
|
||||
private readonly Encoding _encoding;
|
||||
|
||||
public long Position => _position;
|
||||
|
||||
public UnmanagedDataReader(byte* ptr, long size, Dictionary<ulong, string> typesDb = null, Encoding encoding = null)
|
||||
{
|
||||
_encoding = encoding ?? TextEncoding.UTF8;
|
||||
_typesDb = typesDb;
|
||||
_ptr = ptr;
|
||||
_size = size;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadString(bool intern = false) => ReadBool() ? ReadStringRaw(intern) : null;
|
||||
|
||||
public string ReadStringRaw(bool intern = false)
|
||||
{
|
||||
// ReadEncodedInt
|
||||
int length = 0, shift = 0;
|
||||
byte b;
|
||||
|
||||
do
|
||||
{
|
||||
b = *(_ptr + _position++);
|
||||
length |= (b & 0x7F) << shift;
|
||||
shift += 7;
|
||||
}
|
||||
while (b >= 0x80);
|
||||
|
||||
if (length <= 0)
|
||||
{
|
||||
return "".Intern();
|
||||
}
|
||||
|
||||
var str = TextEncoding.GetString(new ReadOnlySpan<byte>(_ptr + _position, length), _encoding);
|
||||
_position += length;
|
||||
return intern ? str.Intern() : str;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public long ReadLong()
|
||||
{
|
||||
var v = BinaryPrimitives.ReadInt64LittleEndian(new ReadOnlySpan<byte>(_ptr + _position, sizeof(long)));
|
||||
_position += sizeof(long);
|
||||
return v;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ulong ReadULong()
|
||||
{
|
||||
var v = BinaryPrimitives.ReadUInt64LittleEndian(new ReadOnlySpan<byte>(_ptr + _position, sizeof(ulong)));
|
||||
_position += sizeof(ulong);
|
||||
return v;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int ReadInt()
|
||||
{
|
||||
var v = BinaryPrimitives.ReadInt32LittleEndian(new ReadOnlySpan<byte>(_ptr + _position, sizeof(int)));
|
||||
_position += sizeof(int);
|
||||
return v;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public uint ReadUInt()
|
||||
{
|
||||
var v = BinaryPrimitives.ReadUInt32LittleEndian(new ReadOnlySpan<byte>(_ptr + _position, sizeof(uint)));
|
||||
_position += sizeof(uint);
|
||||
return v;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public short ReadShort()
|
||||
{
|
||||
var v = BinaryPrimitives.ReadInt16LittleEndian(new ReadOnlySpan<byte>(_ptr + _position, sizeof(short)));
|
||||
_position += sizeof(short);
|
||||
return v;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ushort ReadUShort()
|
||||
{
|
||||
var v = BinaryPrimitives.ReadUInt16LittleEndian(new ReadOnlySpan<byte>(_ptr + _position, sizeof(ushort)));
|
||||
_position += sizeof(ushort);
|
||||
return v;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public double ReadDouble()
|
||||
{
|
||||
var v = BinaryPrimitives.ReadDoubleLittleEndian(new ReadOnlySpan<byte>(_ptr + _position, sizeof(double)));
|
||||
_position += sizeof(double);
|
||||
return v;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public float ReadFloat()
|
||||
{
|
||||
var v = BinaryPrimitives.ReadSingleLittleEndian(new ReadOnlySpan<byte>(_ptr + _position, sizeof(float)));
|
||||
_position += sizeof(float);
|
||||
return v;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte ReadByte() => *(_ptr + _position++);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public sbyte ReadSByte() => (sbyte)ReadByte();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool ReadBool() => ReadByte() != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Serial ReadSerial() => (Serial)ReadUInt();
|
||||
|
||||
public Type ReadType() =>
|
||||
ReadByte() switch
|
||||
{
|
||||
0 => null,
|
||||
1 => AssemblyHandler.FindTypeByFullName(ReadStringRaw()), // Backward compatibility
|
||||
2 => ReadTypeByHash()
|
||||
};
|
||||
|
||||
public Type ReadTypeByHash()
|
||||
{
|
||||
var hash = ReadULong();
|
||||
var t = AssemblyHandler.FindTypeByHash(hash);
|
||||
|
||||
if (t != null)
|
||||
{
|
||||
return t;
|
||||
}
|
||||
|
||||
if (_typesDb == null)
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"The file SerializedTypes.db was not loaded. Type hash '{hash}' could not be found."),
|
||||
"Invalid {Hash} at position {Position}",
|
||||
hash,
|
||||
Position
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!_typesDb.TryGetValue(hash, out var typeName))
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"Type hash '{hash}' is not present in the serialized types database."),
|
||||
"Invalid type hash {Hash} at position {Position}",
|
||||
hash,
|
||||
Position
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
t = AssemblyHandler.FindTypeByFullName(typeName, false);
|
||||
|
||||
if (t == null)
|
||||
{
|
||||
logger.Error(
|
||||
new Exception($"Type '{typeName}' was not found."),
|
||||
"Type {Type} was not found.",
|
||||
typeName
|
||||
);
|
||||
}
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
public int Read(Span<byte> buffer)
|
||||
{
|
||||
var length = buffer.Length;
|
||||
if (length > _size - _position)
|
||||
{
|
||||
throw new OutOfMemoryException();
|
||||
}
|
||||
|
||||
new ReadOnlySpan<byte>(_ptr + _position, length).CopyTo(buffer);
|
||||
_position += length;
|
||||
return length;
|
||||
}
|
||||
|
||||
public virtual long Seek(long offset, SeekOrigin origin)
|
||||
{
|
||||
Debug.Assert(
|
||||
origin != SeekOrigin.End || offset <= 0 && offset > _size,
|
||||
"Attempting to seek to an invalid position using SeekOrigin.End"
|
||||
);
|
||||
Debug.Assert(
|
||||
origin != SeekOrigin.Begin || offset >= 0 && offset < _size,
|
||||
"Attempting to seek to an invalid position using SeekOrigin.Begin"
|
||||
);
|
||||
Debug.Assert(
|
||||
origin != SeekOrigin.Current || _position + offset >= 0 && _position + offset < _size,
|
||||
"Attempting to seek to an invalid position using SeekOrigin.Current"
|
||||
);
|
||||
|
||||
var position = Math.Max(0L, origin switch
|
||||
{
|
||||
SeekOrigin.Current => _position + offset,
|
||||
SeekOrigin.End => _size + offset,
|
||||
_ => offset // Begin
|
||||
});
|
||||
|
||||
_position = position;
|
||||
return _position;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue