fix: Cleans up entity persistence. Generalizes Mobiles/Items/Guilds (#1528)

### Summary
- [X] Fixed a bug where entity persistence was serialized out of order, causing world corruption.
- [X] Fixed LastSerialized not being utilized properly and dangling references still becoming an issue.
- [X] Added a new `GenericEntityPersistence<T>` type to encapsulate `ISerializable` serialization.
- [X] Removing the custom logic and moved Items, Mobiles, Guilds, and Accounts  to GenericEntityPersistence.
- [X] Changed serialization to use the singleton pattern to reduce calling methods from stored variables.
This commit is contained in:
Kamron Batman 2023-10-01 22:10:47 -07:00 committed by GitHub
parent e0225c6e59
commit f2de2fbb77
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
34 changed files with 721 additions and 970 deletions

View file

@ -19,8 +19,6 @@ using System.Collections.Generic;
using System.IO;
using System.IO.MemoryMappedFiles;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
namespace Server;
@ -72,12 +70,6 @@ public static class EntityPersistence
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void SaveEntities<T>(
IEnumerable<T> list,
Action<T> serializer
) where T : class, ISerializable => Parallel.ForEach(list, serializer);
public static Dictionary<I, T> LoadIndex<I, T>(
string path,
IIndexInfo<I> indexInfo,
@ -176,6 +168,7 @@ public static class EntityPersistence
}
idxReader.Close();
entities.TrimExcess();
return map;
}

View file

@ -17,10 +17,7 @@ using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using Server.Guilds;
using Server.Logging;
@ -41,120 +38,34 @@ public static class World
{
private static ILogger logger = LogFactory.GetLogger(typeof(World));
private static ItemPersistence _itemPersistence = new();
private static MobilePersistence _mobilePersistence = new();
private static GenericEntityPersistence<BaseGuild> _guildPersistence = new("Guilds", 3, 1, 0x7FFFFFFF);
private static ManualResetEvent m_DiskWriteHandle = new(true);
private static Dictionary<Serial, IEntity> _pendingAdd = new();
private static Dictionary<Serial, IEntity> _pendingDelete = new();
private static ConcurrentQueue<Item> _decayQueue = new();
private static string _tempSavePath; // Path to the temporary folder for the save
private static bool _enableSaveStats;
public const bool DirtyTrackingEnabled = false;
public const uint ItemOffset = 0x40000000;
public const uint MaxItemSerial = 0x7FFFFFFF;
public const uint MaxMobileSerial = ItemOffset - 1;
private const uint _maxItems = MaxItemSerial - ItemOffset + 1;
private static Serial _lastMobile = Serial.Zero;
private static Serial _lastItem = (Serial)ItemOffset;
private static Serial _lastGuild = Serial.Zero;
public static Serial NewMobile => _mobilePersistence.NewEntity;
public static Serial NewItem => _itemPersistence.NewEntity;
public static Serial NewGuild => _guildPersistence.NewEntity;
public static Serial NewMobile
{
get
{
#if THREADGUARD
if (Thread.CurrentThread != Core.Thread)
{
logger.Error(
"Attempted to get a new mobile serial from the wrong thread!\n{StackTrace}",
new StackTrace()
);
}
#endif
var last = _lastMobile;
var maxMobile = (Serial)MaxMobileSerial;
for (int i = 0; i < MaxMobileSerial; i++)
{
last++;
if (last > maxMobile)
{
last = (Serial)1;
}
if (FindMobile(last, true) == null)
{
return _lastMobile = last;
}
}
OutOfMemory("No serials left to allocate for mobiles");
return Serial.MinusOne;
}
}
public static Serial NewItem
{
get
{
#if THREADGUARD
if (Thread.CurrentThread != Core.Thread)
{
logger.Error(
"Attempted to get a new item serial from the wrong thread!\n{StackTrace}",
new StackTrace()
);
}
#endif
var last = _lastItem;
for (int i = 0; i < _maxItems; i++)
{
last++;
if (last > MaxItemSerial)
{
last = (Serial)ItemOffset;
}
if (FindItem(last, true) == null)
{
return _lastItem = last;
}
}
OutOfMemory("No serials left to allocate for items");
return Serial.MinusOne;
}
}
public static Serial NewGuild
{
get
{
while (FindGuild(_lastGuild += 1) != null)
{
}
return _lastGuild;
}
}
private static void OutOfMemory(string message) => throw new OutOfMemoryException(message);
public static Dictionary<Serial, Item> Items => _itemPersistence.EntitiesBySerial;
public static Dictionary<Serial, Mobile> Mobiles => _mobilePersistence.EntitiesBySerial;
public static Dictionary<Serial, BaseGuild> Guilds => _guildPersistence.EntitiesBySerial;
public static string SavePath { get; private set; }
public static WorldState WorldState { get; private set; }
public static bool Saving => WorldState == WorldState.Saving;
public static bool Running => WorldState is not WorldState.Loading and not WorldState.Initial;
public static bool Loading => WorldState == WorldState.Loading;
public static Dictionary<Serial, Mobile> Mobiles { get; private set; }
public static Dictionary<Serial, Item> Items { get; private set; }
public static Dictionary<Serial, BaseGuild> Guilds { get; private set; }
public static void Configure()
{
var tempSavePath = ServerConfiguration.GetSetting("world.tempSavePath", "temp");
@ -162,11 +73,6 @@ public static class World
var savePath = ServerConfiguration.GetOrUpdateSetting("world.savePath", "Saves");
SavePath = PathUtility.GetFullPath(savePath);
_enableSaveStats = ServerConfiguration.GetOrUpdateSetting("world.enableSaveStats", false);
// Mobiles & Items
Persistence.Register("Mobiles & Items", SaveEntities, WriteEntities, LoadEntities, 1);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@ -243,36 +149,6 @@ public static class World
NetState.FlushAll();
}
internal static void LoadEntities(string basePath, Dictionary<ulong, string> typesDb)
{
IIndexInfo<Serial> itemIndexInfo = new EntityTypeIndex("Items");
IIndexInfo<Serial> mobileIndexInfo = new EntityTypeIndex("Mobiles");
IIndexInfo<Serial> guildIndexInfo = new EntityTypeIndex("Guilds");
Mobiles = EntityPersistence.LoadIndex(basePath, mobileIndexInfo, typesDb, out List<EntitySpan<Mobile>> mobiles);
Items = EntityPersistence.LoadIndex(basePath, itemIndexInfo, typesDb, out List<EntitySpan<Item>> items);
Guilds = EntityPersistence.LoadIndex(basePath, guildIndexInfo, typesDb, out List<EntitySpan<BaseGuild>> guilds);
if (Mobiles.Count > 0)
{
_lastMobile = Mobiles.Keys.Max();
}
if (Items.Count > 0)
{
_lastItem = Items.Keys.Max();
}
if (Guilds.Count > 0)
{
_lastGuild = Guilds.Keys.Max();
}
EntityPersistence.LoadData(basePath, mobileIndexInfo, typesDb, mobiles);
EntityPersistence.LoadData(basePath, itemIndexInfo, typesDb, items);
EntityPersistence.LoadData(basePath, guildIndexInfo, typesDb, guilds);
}
public static void Load()
{
if (WorldState != WorldState.Initial)
@ -291,25 +167,7 @@ public static class World
// Set the world to running before we process our queues
WorldState = WorldState.Running;
ProcessSafetyQueues();
foreach (var item in Items.Values)
{
if (item.Parent == null)
{
item.UpdateTotals();
}
item.ClearProperties();
}
foreach (var m in Mobiles.Values)
{
m.UpdateRegion(); // Is this really needed?
m.UpdateTotals();
m.ClearProperties();
}
Persistence.PostDeserializeAll(); // Process safety queues
watch.Stop();
@ -321,102 +179,13 @@ public static class World
);
}
private static void ProcessSafetyQueues()
{
foreach (var entity in _pendingAdd.Values)
{
AddEntity(entity);
}
_pendingAdd.Clear();
foreach (var entity in _pendingDelete.Values)
{
if (_pendingAdd.ContainsKey(entity.Serial))
{
logger.Warning("Entity {Entity} was both pending deletion and addition after save", entity);
}
RemoveEntity(entity);
}
_pendingDelete.Clear();
}
private static void AppendSafetyLog(string action, ISerializable entity)
{
var message =
$"Warning: Attempted to {{Action}} {{Entity}} during world save.{Environment.NewLine}This action could cause inconsistent state.{Environment.NewLine}It is strongly advised that the offending scripts be corrected.";
logger.Information(message, action, entity);
try
{
using var op = new StreamWriter("world-save-errors.log", true);
op.WriteLine("{0}\t{1}", DateTime.UtcNow, message);
op.WriteLine(new StackTrace(2).ToString());
op.WriteLine();
}
catch
{
// ignored
}
}
private static void FinishWorldSave()
{
WorldState = WorldState.Running;
ProcessDecay();
ProcessSafetyQueues();
}
private static void TraceSave(params IEnumerable<KeyValuePair<string, int>>[] entityTypes)
{
try
{
int count = 0;
var timestamp = Utility.GetTimeStamp();
var saveStatsPath = Path.Combine(Core.BaseDirectory, $"Logs/Saves/Save-Stats-{timestamp}.log");
PathUtility.EnsureDirectory(saveStatsPath);
using var op = new StreamWriter(saveStatsPath, true);
for (var i = 0; i < entityTypes.Length; i++)
{
foreach (var (t, c) in entityTypes[i])
{
op.WriteLine("{0}: {1}", t, c);
count++;
}
}
op.WriteLine("- Total: {0}", count);
op.WriteLine();
op.WriteLine();
}
catch
{
// ignored
}
}
internal static void WriteEntities(string basePath)
{
IIndexInfo<Serial> itemIndexInfo = new EntityTypeIndex("Items");
IIndexInfo<Serial> mobileIndexInfo = new EntityTypeIndex("Mobiles");
IIndexInfo<Serial> guildIndexInfo = new EntityTypeIndex("Guilds");
EntityPersistence.WriteEntities(mobileIndexInfo, Mobiles, basePath, SerializedTypes, out var mobileCounts);
EntityPersistence.WriteEntities(itemIndexInfo, Items, basePath, SerializedTypes, out var itemCounts);
EntityPersistence.WriteEntities(guildIndexInfo, Guilds, basePath, SerializedTypes, out var guildCounts);
if (_enableSaveStats)
{
TraceSave(mobileCounts?.ToList(), itemCounts?.ToList(), guildCounts?.ToList());
}
Persistence.PostSerializeAll(); // Process safety queues
}
public static void WriteFiles(object state)
@ -516,25 +285,6 @@ public static class World
*/
public static ConcurrentQueue<Type> SerializedTypes { get; } = new();
private static void SaveEntities()
{
_serializationStart = DateTime.UtcNow;
EntityPersistence.SaveEntities(Items.Values, SaveEntity);
EntityPersistence.SaveEntities(Mobiles.Values, SaveEntity);
EntityPersistence.SaveEntities(Guilds.Values, SaveEntity);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void SaveEntity<T>(T entity) where T : class, ISerializable
{
if (entity is Item item && item.CanDecay() && item.LastMoved + item.DecayTime <= _serializationStart)
{
EnqueueForDecay(item);
}
entity.Serialize(SerializedTypes);
}
public static void Save()
{
if (WorldState != WorldState.Running)
@ -558,7 +308,8 @@ public static class World
try
{
Persistence.Serialize();
_serializationStart = Core.Now;
Persistence.SerializeAll();
EventSink.InvokeWorldSave();
}
catch (Exception ex)
@ -593,231 +344,119 @@ public static class World
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static IEntity FindEntity(Serial serial, bool returnDeleted = false, bool returnPending = true) =>
FindEntity<IEntity>(serial, returnDeleted, returnPending);
public static T FindEntity<T>(Serial serial, bool returnDeleted = false, bool returnPending = true) where T : class, IEntity
{
switch (WorldState)
{
default: return default;
case WorldState.Loading:
case WorldState.Saving:
case WorldState.WritingSave:
{
if (returnDeleted && returnPending && _pendingDelete.TryGetValue(serial, out var entity))
{
return entity as T;
}
if (!returnPending || !_pendingAdd.TryGetValue(serial, out entity))
{
if (serial.IsItem)
{
if (Items.TryGetValue(serial, out var item))
{
return item as T;
}
}
else // if (serial.IsMobile)
{
if (Mobiles.TryGetValue(serial, out var mob))
{
return mob as T;
}
}
}
return null;
}
case WorldState.Running:
{
if (serial.IsItem)
{
return Items.TryGetValue(serial, out var item) ? item as T : null;
}
if (serial.IsMobile)
{
return Mobiles.TryGetValue(serial, out var mob) ? mob as T : null;
}
return default;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Item FindItem(Serial serial, bool returnDeleted = false) => FindEntity<Item>(serial, returnDeleted);
public static Item FindItem(Serial serial, bool returnDeleted = false) => _itemPersistence.Find(serial, returnDeleted);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Mobile FindMobile(Serial serial, bool returnDeleted = false) =>
FindEntity<Mobile>(serial, returnDeleted);
_mobilePersistence.Find(serial, returnDeleted);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static BaseGuild FindGuild(Serial serial) => Guilds.TryGetValue(serial, out var guild) ? guild : null;
public static void AddEntity<T>(T entity) where T : class, IEntity
// Legacy: Only used for retrieving Items and Mobiles.
public static void AddEntity(IEntity entity)
{
switch (WorldState)
if (entity is Item item)
{
default: // Not Running
{
throw new Exception($"Added {entity.GetType().Name} before world load.");
}
case WorldState.Saving:
{
AppendSafetyLog("add", entity);
goto case WorldState.WritingSave;
}
case WorldState.Loading:
case WorldState.WritingSave:
{
if (_pendingDelete.Remove(entity.Serial))
{
logger.Warning("Deleted then added {Entity} during {WorldState} state.", entity.GetType().Name, WorldState.ToString());
}
_pendingAdd[entity.Serial] = entity;
break;
}
case WorldState.Running:
{
if (entity.Serial.IsItem)
{
ref var item = ref CollectionsMarshal.GetValueRefOrAddDefault(Items, entity.Serial, out bool exists);
if (exists)
{
if (item == entity)
{
logger.Error(
$"Attempted to add '{{Entity}}' ({{Serial}}) to World.Items but it already exists in the collection.{Environment.NewLine}{{StackTrace}}",
entity.GetType().FullName,
entity.Serial,
new StackTrace()
);
}
else
{
logger.Error(
$"Attempted to add '{{Entity}}' ({{Serial}}) to World.Items but found '{{ExistingEntity}}' ({{ExistingSerial}}).{Environment.NewLine}{{StackTrace}}",
entity.GetType().FullName,
entity.Serial,
item.GetType().FullName,
item.Serial,
new StackTrace()
);
}
}
else
{
item = entity as Item;
}
}
if (entity.Serial.IsMobile)
{
ref var mob = ref CollectionsMarshal.GetValueRefOrAddDefault(Mobiles, entity.Serial, out bool exists);
if (exists)
{
if (mob == entity)
{
logger.Error(
$"Attempted to add '{{Entity}}' ({{Serial}}) to World.Mobiles but it already exists in the collection.{Environment.NewLine}{{StackTrace}}",
entity.GetType().FullName,
entity.Serial,
new StackTrace()
);
}
else
{
logger.Error(
$"Attempted to add '{{Entity}}' ({{Serial}}) to World.Mobiles but found '{{ExistingEntity}}' ({{ExistingSerial}}).{Environment.NewLine}{{StackTrace}}",
entity.GetType().FullName,
entity.Serial,
mob.GetType().FullName,
mob.Serial,
new StackTrace()
);
}
}
else
{
mob = entity as Mobile;
}
}
break;
}
_itemPersistence.AddEntity(item);
}
}
public static void AddGuild(BaseGuild entity)
{
ref var guild = ref CollectionsMarshal.GetValueRefOrAddDefault(Guilds, entity.Serial, out bool exists);
if (exists)
else if (entity is Mobile mobile)
{
if (guild == entity)
{
logger.Error(
$"Attempted to add '{{Entity}}' ({{Serial}}) to World.Guilds but it already exists in the collection.{Environment.NewLine}{{StackTrace}}",
entity.GetType().FullName,
entity.Serial,
new StackTrace()
);
}
else
{
logger.Error(
$"Attempted to add '{{Entity}}' ({{Serial}}) to World.Guilds but found '{{ExistingEntity}}' ({{ExistingSerial}}).{Environment.NewLine}{{StackTrace}}",
entity.GetType().FullName,
entity.Serial,
guild.GetType().FullName,
guild.Serial,
new StackTrace()
);
}
_mobilePersistence.AddEntity(mobile);
}
else
{
guild = entity;
logger.Warning($"Attempted to call World.AddEntity with '{entity.GetType()}'. Must be a mobile or item.");
}
}
public static void RemoveEntity<T>(T entity) where T : class, IEntity
public static void RemoveEntity(IEntity entity)
{
switch (WorldState)
if (entity is Item item)
{
default: // Not Running
{
throw new Exception($"Removed {entity.GetType().Name} before world load.");
}
case WorldState.Saving:
{
AppendSafetyLog("delete", entity);
goto case WorldState.WritingSave;
}
case WorldState.Loading:
case WorldState.WritingSave:
{
_pendingAdd.Remove(entity.Serial);
_pendingDelete[entity.Serial] = entity;
break;
}
case WorldState.Running:
{
if (entity.Serial.IsItem)
{
Items.Remove(entity.Serial);
}
if (entity.Serial.IsMobile)
{
Mobiles.Remove(entity.Serial);
}
break;
}
_itemPersistence.RemoveEntity(item);
}
else if (entity is Mobile mobile)
{
_mobilePersistence.RemoveEntity(mobile);
}
else
{
logger.Warning($"Attempted to call World.RemoveEntity with '{entity.GetType()}'. Must be a mobile or item.");
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void RemoveGuild(BaseGuild guild) => Guilds.Remove(guild.Serial);
public static BaseGuild FindGuild(Serial serial) => _guildPersistence.Find(serial);
public static void AddGuild(BaseGuild guild) => _guildPersistence.AddEntity(guild);
public static void RemoveGuild(BaseGuild guild) => _guildPersistence.RemoveEntity(guild);
// Legacy: Only used for retrieving Items and Mobiles.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static IEntity FindEntity(Serial serial, bool returnDeleted = false, bool returnPending = false) =>
FindEntity<IEntity>(serial, returnDeleted, returnPending);
// Legacy: Only used for retrieving Items and Mobiles.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T FindEntity<T>(Serial serial, bool returnDeleted = false, bool returnPending = false)
where T : class, IEntity
{
if (serial.IsItem)
{
return _itemPersistence.Find(serial, returnDeleted, returnPending) as T;
}
return _mobilePersistence.Find(serial, returnDeleted, returnPending) as T;
}
private class ItemPersistence : GenericEntityPersistence<Item>
{
public ItemPersistence() : base("Items", 2, ItemOffset, MaxItemSerial)
{
}
protected override void SerializeEntity(Item item)
{
if (item.CanDecay() && item.LastMoved + item.DecayTime <= _serializationStart)
{
EnqueueForDecay(item);
}
((ISerializable)item).Serialize(SerializedTypes);
}
public override void PostDeserialize()
{
base.PostDeserialize();
foreach (var item in EntitiesBySerial.Values)
{
if (item.Parent == null)
{
item.UpdateTotals();
}
item.ClearProperties();
}
}
}
private class MobilePersistence : GenericEntityPersistence<Mobile>
{
public MobilePersistence() : base("Mobiles", 1, 1, MaxMobileSerial)
{
}
public override void PostDeserialize()
{
base.PostDeserialize();
foreach (var m in EntitiesBySerial.Values)
{
m.UpdateRegion(); // Is this really needed?
m.UpdateTotals();
m.ClearProperties();
}
}
}
}