From 67af95cf0b686f3d27d721bd352d58d6eabb4693 Mon Sep 17 00:00:00 2001 From: asayre Date: Mon, 20 Dec 2010 03:34:25 +0000 Subject: [PATCH] Fix for RunUO Crash if Performance tab on Admin gump is open within first 100seconds of server running. Completion of the DynamicSaveStrategy. Background saves are now safe; triggered only by the autoSave or command [BackgroundSave ([BGSave) --- Scripts/Commands/Handlers.cs | 13 +++- Scripts/Misc/AutoSave.cs | 15 +++- Scripts/Misc/CrashGuard.cs | 7 +- Server/Main.cs | 6 +- Server/Persistence/DualSaveStrategy.cs | 14 +++- Server/Persistence/DynamicSaveStrategy.cs | 24 +++--- Server/Persistence/ParallelSaveStrategy.cs | 5 +- Server/Persistence/QueuedMemoryWriter.cs | 91 +++++++++++----------- Server/Persistence/SaveStrategy.cs | 2 +- Server/Persistence/StandardSaveStrategy.cs | 37 ++++++--- Server/Serialization.cs | 4 + Server/World.cs | 35 +++++++-- 12 files changed, 169 insertions(+), 84 deletions(-) diff --git a/Scripts/Commands/Handlers.cs b/Scripts/Commands/Handlers.cs index 50ce98614..261c3fdc3 100644 --- a/Scripts/Commands/Handlers.cs +++ b/Scripts/Commands/Handlers.cs @@ -48,6 +48,9 @@ namespace Server.Commands Register( "Help", AccessLevel.Player, new CommandEventHandler( Help_OnCommand ) ); Register( "Save", AccessLevel.Administrator, new CommandEventHandler( Save_OnCommand ) ); + Register( "BackgroundSave", AccessLevel.Administrator, new CommandEventHandler( BackgroundSave_OnCommand ) ); + Register( "BGSave", AccessLevel.Administrator, new CommandEventHandler( BackgroundSave_OnCommand ) ); + Register( "SaveBG", AccessLevel.Administrator, new CommandEventHandler( BackgroundSave_OnCommand ) ); Register( "Move", AccessLevel.GameMaster, new CommandEventHandler( Move_OnCommand ) ); Register( "Client", AccessLevel.Counselor, new CommandEventHandler( Client_OnCommand ) ); @@ -55,7 +58,7 @@ namespace Server.Commands Register( "SMsg", AccessLevel.Counselor, new CommandEventHandler( StaffMessage_OnCommand ) ); Register( "SM", AccessLevel.Counselor, new CommandEventHandler( StaffMessage_OnCommand ) ); Register( "S", AccessLevel.Counselor, new CommandEventHandler( StaffMessage_OnCommand ) ); - + Register( "BCast", AccessLevel.GameMaster, new CommandEventHandler( BroadcastMessage_OnCommand ) ); Register( "BC", AccessLevel.GameMaster, new CommandEventHandler( BroadcastMessage_OnCommand ) ); Register( "B", AccessLevel.GameMaster, new CommandEventHandler( BroadcastMessage_OnCommand ) ); @@ -639,6 +642,14 @@ namespace Server.Commands Misc.AutoSave.Save(); } + [Usage("BackgroundSave")] + [Aliases("BGSave", "SaveBG")] + [Description("Saves the world, writing to the disk in the background")] + private static void BackgroundSave_OnCommand(CommandEventArgs e) + { + Misc.AutoSave.Save( true ); + } + private static bool FixMap( ref Map map, ref Point3D loc, Item item ) { if ( map == null || map == Map.Internal ) diff --git a/Scripts/Misc/AutoSave.cs b/Scripts/Misc/AutoSave.cs index ef1b2bdac..349d4ba06 100644 --- a/Scripts/Misc/AutoSave.cs +++ b/Scripts/Misc/AutoSave.cs @@ -52,7 +52,7 @@ namespace Server.Misc if ( m_Warning == TimeSpan.Zero ) { - Save(); + Save( true ); } else { @@ -72,14 +72,21 @@ namespace Server.Misc } public static void Save() + { + AutoSave.Save( false ); + } + + public static void Save( bool permitBackgroundWrite ) { if ( AutoRestart.Restarting ) return; - try{ Backup(); } - catch (Exception e) { Console.WriteLine("WARNING: Automatic backup FAILED: {0}", e); } + World.WaitForWriteCompletion(); - World.Save(); + try{ Backup(); } + catch ( Exception e ) { Console.WriteLine("WARNING: Automatic backup FAILED: {0}", e); } + + World.Save( true, permitBackgroundWrite ); } private static string[] m_Backups = new string[] diff --git a/Scripts/Misc/CrashGuard.cs b/Scripts/Misc/CrashGuard.cs index fc7d15eef..e6c64454b 100644 --- a/Scripts/Misc/CrashGuard.cs +++ b/Scripts/Misc/CrashGuard.cs @@ -25,11 +25,14 @@ namespace Server.Misc public static void CrashGuard_OnCrash( CrashedEventArgs e ) { + if ( GenerateReport ) + GenerateCrashReport( e ); + + World.WaitForWriteCompletion(); + if ( SaveBackup ) Backup(); - if ( GenerateReport ) - GenerateCrashReport( e ); /*if ( Core.Service ) e.Close = true; diff --git a/Server/Main.cs b/Server/Main.cs index f1e1d13c0..fed00f162 100644 --- a/Server/Main.cs +++ b/Server/Main.cs @@ -302,7 +302,7 @@ namespace Server if( World.Saving || ( m_Service && type == ConsoleEventType.CTRL_LOGOFF_EVENT ) ) return true; - Kill(); + Kill(); //Kill -> HandleClosed will hadnle waiting for the completion of flushign to disk return true; } @@ -315,7 +315,7 @@ namespace Server private static bool m_Closing; public static bool Closing { get { return m_Closing; } } - private static long m_CycleIndex; + private static long m_CycleIndex = 1; private static float[] m_CyclesPerSecond = new float[100]; public static float CyclesPerSecond @@ -364,6 +364,8 @@ namespace Server Console.Write( "Exiting..." ); + World.WaitForWriteCompletion(); + if( !m_Crashed ) EventSink.InvokeShutdown( new ShutdownEventArgs() ); diff --git a/Server/Persistence/DualSaveStrategy.cs b/Server/Persistence/DualSaveStrategy.cs index 7590925c6..c9a4c9ae6 100644 --- a/Server/Persistence/DualSaveStrategy.cs +++ b/Server/Persistence/DualSaveStrategy.cs @@ -37,18 +37,24 @@ namespace Server { public DualSaveStrategy() { } - public override void Save( SaveMetrics metrics ) { + public override void Save( SaveMetrics metrics, bool permitBackgroundWrite ) + { + this.PermitBackgroundWrite = permitBackgroundWrite; + Thread saveThread = new Thread( delegate() { - SaveItems( metrics ); + SaveItems(metrics); } ); saveThread.Name = "Item Save Subset"; saveThread.Start(); - SaveMobiles( metrics ); - SaveGuilds( metrics ); + SaveMobiles(metrics); + SaveGuilds(metrics); saveThread.Join(); + + if (UseSequentialWriters) + World.NotifyDiskWriteComplete(); } } } \ No newline at end of file diff --git a/Server/Persistence/DynamicSaveStrategy.cs b/Server/Persistence/DynamicSaveStrategy.cs index 7b4afee9c..79cc8594f 100644 --- a/Server/Persistence/DynamicSaveStrategy.cs +++ b/Server/Persistence/DynamicSaveStrategy.cs @@ -59,7 +59,7 @@ namespace Server _guildThreadWriters = new BlockingCollection(); } - public override void Save(SaveMetrics metrics) + public override void Save(SaveMetrics metrics, bool permitBackgroundWrite) { this._metrics = metrics; @@ -73,13 +73,21 @@ namespace Server SaveTypeDatabases(); + if (permitBackgroundWrite) + { + //This option makes it finish the writing to disk in the background, continuing even after Save() returns. + Task.Factory.ContinueWhenAll(saveTasks, _ => + { + CloseFiles(); - Task.WaitAll(saveTasks); //Waits for the completion of all of the tasks(committing to disk) - CloseFiles(); - - //This option makes it finish the writing to disk in the background, continuing even after Save() returns. VERY DANGEROUS. - //TODO: Make this less dangerous, by notifying rest of the server that this can and does happen - //Task.Factory.ContinueWhenAll(saveTasks, _ => CloseFiles()); + World.NotifyDiskWriteComplete(); + }); + } + else + { + Task.WaitAll(saveTasks); //Waits for the completion of all of the tasks(committing to disk) + CloseFiles(); + } } private Task StartCommitTask(BlockingCollection threadWriter, SequentialFileWriter data, SequentialFileWriter index) @@ -264,8 +272,6 @@ namespace Server _guildData.Close(); _guildIndex.Close(); - - Console.WriteLine("Closing files"); } private void WriteCount(SequentialFileWriter indexFile, int count) diff --git a/Server/Persistence/ParallelSaveStrategy.cs b/Server/Persistence/ParallelSaveStrategy.cs index 53666b99b..11b1c25be 100644 --- a/Server/Persistence/ParallelSaveStrategy.cs +++ b/Server/Persistence/ParallelSaveStrategy.cs @@ -60,7 +60,8 @@ namespace Server { private bool finished; - public override void Save( SaveMetrics metrics ) { + public override void Save(SaveMetrics metrics, bool permitBackgroundWrite) + { this.metrics = metrics; OpenFiles(); @@ -163,6 +164,8 @@ namespace Server { guildData.Close(); guildIndex.Close(); + + World.NotifyDiskWriteComplete(); } private void OnSerialized( ConsumableEntry entry ) { diff --git a/Server/Persistence/QueuedMemoryWriter.cs b/Server/Persistence/QueuedMemoryWriter.cs index 3bded9e4a..82cc1aa50 100644 --- a/Server/Persistence/QueuedMemoryWriter.cs +++ b/Server/Persistence/QueuedMemoryWriter.cs @@ -25,7 +25,6 @@ using System.Text; namespace Server { - public sealed class QueuedMemoryWriter : BinaryFileWriter { private struct IndexInfo @@ -36,11 +35,11 @@ namespace Server } private MemoryStream _memStream; - private Queue _indexQueue = new Queue(); //TODO: Pick a more optimal starting size + private List _orderedIndexInfo = new List(); protected override int BufferSize { - get { return 4096; } + get { return 512; } } public QueuedMemoryWriter() @@ -58,66 +57,70 @@ namespace Server info.typeCode = serializable.TypeReference; //For guilds, this will automagically be zero. info.serial = serializable.SerialIdentity; - _indexQueue.Enqueue(info); + _orderedIndexInfo.Add(info); } public void CommitTo(SequentialFileWriter dataFile, SequentialFileWriter indexFile) { this.Flush(); - byte[] memBuffer = _memStream.GetBuffer(); int memLength = (int)_memStream.Position; - long actualPosition = dataFile.Position; - - dataFile.Write(memBuffer, 0, memLength); //The buffer contains the data from many items. - - //Console.WriteLine("Writing {0} bytes starting at {1}", memLength, actualPosition); - - byte[] indexBuffer = new byte[20]; - - //int indexWritten = _indexQueue.Count * indexBuffer.Length; - //int totalWritten = memLength + indexWritten - - while (_indexQueue.Count > 0) + if (memLength > 0) { - IndexInfo info = _indexQueue.Dequeue(); + byte[] memBuffer = _memStream.GetBuffer(); - int typeCode = info.typeCode; - int serial = info.serial; - int length = info.size; + long actualPosition = dataFile.Position; + + dataFile.Write(memBuffer, 0, memLength); //The buffer contains the data from many items. + + //Console.WriteLine("Writing {0} bytes starting at {1}, with {2} things", memLength, actualPosition, _orderedIndexInfo.Count); + + byte[] indexBuffer = new byte[20]; + + //int indexWritten = _orderedIndexInfo.Count * indexBuffer.Length; + //int totalWritten = memLength + indexWritten + + for (int i = 0; i < _orderedIndexInfo.Count; i++) + { + IndexInfo info = _orderedIndexInfo[i]; + + int typeCode = info.typeCode; + int serial = info.serial; + int length = info.size; - indexBuffer[0] = (byte)(info.typeCode); - indexBuffer[1] = (byte)(info.typeCode >> 8); - indexBuffer[2] = (byte)(info.typeCode >> 16); - indexBuffer[3] = (byte)(info.typeCode >> 24); + indexBuffer[0] = (byte)(info.typeCode); + indexBuffer[1] = (byte)(info.typeCode >> 8); + indexBuffer[2] = (byte)(info.typeCode >> 16); + indexBuffer[3] = (byte)(info.typeCode >> 24); - indexBuffer[4] = (byte)(info.serial); - indexBuffer[5] = (byte)(info.serial >> 8); - indexBuffer[6] = (byte)(info.serial >> 16); - indexBuffer[7] = (byte)(info.serial >> 24); + indexBuffer[4] = (byte)(info.serial); + indexBuffer[5] = (byte)(info.serial >> 8); + indexBuffer[6] = (byte)(info.serial >> 16); + indexBuffer[7] = (byte)(info.serial >> 24); - indexBuffer[8] = (byte)(actualPosition); - indexBuffer[9] = (byte)(actualPosition >> 8); - indexBuffer[10] = (byte)(actualPosition >> 16); - indexBuffer[11] = (byte)(actualPosition >> 24); - indexBuffer[12] = (byte)(actualPosition >> 32); - indexBuffer[13] = (byte)(actualPosition >> 40); - indexBuffer[14] = (byte)(actualPosition >> 48); - indexBuffer[15] = (byte)(actualPosition >> 56); + indexBuffer[8] = (byte)(actualPosition); + indexBuffer[9] = (byte)(actualPosition >> 8); + indexBuffer[10] = (byte)(actualPosition >> 16); + indexBuffer[11] = (byte)(actualPosition >> 24); + indexBuffer[12] = (byte)(actualPosition >> 32); + indexBuffer[13] = (byte)(actualPosition >> 40); + indexBuffer[14] = (byte)(actualPosition >> 48); + indexBuffer[15] = (byte)(actualPosition >> 56); - indexBuffer[16] = (byte)(info.size); - indexBuffer[17] = (byte)(info.size >> 8); - indexBuffer[18] = (byte)(info.size >> 16); - indexBuffer[19] = (byte)(info.size >> 24); + indexBuffer[16] = (byte)(info.size); + indexBuffer[17] = (byte)(info.size >> 8); + indexBuffer[18] = (byte)(info.size >> 16); + indexBuffer[19] = (byte)(info.size >> 24); - indexFile.Write(indexBuffer, 0, indexBuffer.Length); + indexFile.Write(indexBuffer, 0, indexBuffer.Length); - actualPosition += info.size; + actualPosition += info.size; + } } - this.Close(); //We're donezo with this writer. + this.Close(); //We're done with this writer. } } } \ No newline at end of file diff --git a/Server/Persistence/SaveStrategy.cs b/Server/Persistence/SaveStrategy.cs index 3feb803ab..6d91da9b7 100644 --- a/Server/Persistence/SaveStrategy.cs +++ b/Server/Persistence/SaveStrategy.cs @@ -51,7 +51,7 @@ namespace Server } public abstract string Name { get; } - public abstract void Save(SaveMetrics metrics); + public abstract void Save(SaveMetrics metrics, bool permitBackgroundWrite); public abstract void ProcessDecay(); } diff --git a/Server/Persistence/StandardSaveStrategy.cs b/Server/Persistence/StandardSaveStrategy.cs index 171f0e028..134020722 100644 --- a/Server/Persistence/StandardSaveStrategy.cs +++ b/Server/Persistence/StandardSaveStrategy.cs @@ -35,25 +35,38 @@ namespace Server { } private Queue _decayQueue; + private bool _permitBackgroundWrite; public StandardSaveStrategy() { _decayQueue = new Queue(); } - public override void Save( SaveMetrics metrics ) { - SaveMobiles( metrics ); - SaveItems( metrics ); - SaveGuilds( metrics ); + protected bool PermitBackgroundWrite { get { return _permitBackgroundWrite; } set { _permitBackgroundWrite = value; } } + + protected bool UseSequentialWriters { get { return (World.SaveType == World.SaveOption.Normal || !_permitBackgroundWrite); } } + + public override void Save(SaveMetrics metrics, bool permitBackgroundWrite) + { + _permitBackgroundWrite = permitBackgroundWrite; + + SaveMobiles(metrics); + SaveItems(metrics); + SaveGuilds(metrics); + + if (UseSequentialWriters) + World.NotifyDiskWriteComplete(); } - protected void SaveMobiles( SaveMetrics metrics ) { + protected void SaveMobiles(SaveMetrics metrics) + { Dictionary mobiles = World.Mobiles; GenericWriter idx; GenericWriter tdb; GenericWriter bin; - if ( World.SaveType == World.SaveOption.Normal ) { + if (UseSequentialWriters) + { idx = new BinaryFileWriter( World.MobileIndexPath, false ); tdb = new BinaryFileWriter( World.MobileTypesPath, false ); bin = new BinaryFileWriter( World.MobileDataPath, true ); @@ -92,7 +105,8 @@ namespace Server { bin.Close(); } - protected void SaveItems( SaveMetrics metrics ) { + protected void SaveItems(SaveMetrics metrics) + { Dictionary items = World.Items; List decaying = new List(); @@ -100,7 +114,8 @@ namespace Server { GenericWriter tdb; GenericWriter bin; - if ( World.SaveType == World.SaveOption.Normal ) { + if (UseSequentialWriters) + { idx = new BinaryFileWriter( World.ItemIndexPath, false ); tdb = new BinaryFileWriter( World.ItemTypesPath, false ); bin = new BinaryFileWriter( World.ItemDataPath, true ); @@ -142,11 +157,13 @@ namespace Server { bin.Close(); } - protected void SaveGuilds( SaveMetrics metrics ) { + protected void SaveGuilds(SaveMetrics metrics) + { GenericWriter idx; GenericWriter bin; - if ( World.SaveType == World.SaveOption.Normal ) { + if (UseSequentialWriters) + { idx = new BinaryFileWriter( World.GuildIndexPath, false ); bin = new BinaryFileWriter( World.GuildDataPath, true ); } else { diff --git a/Server/Serialization.cs b/Server/Serialization.cs index 35b799f86..016c31b39 100644 --- a/Server/Serialization.cs +++ b/Server/Serialization.cs @@ -1217,7 +1217,11 @@ namespace Server if( m_Owner.m_Closed ) m_Owner.m_File.Close(); + AsyncWriter.m_ThreadCount--; + + if (AsyncWriter.m_ThreadCount <= 0) + World.NotifyDiskWriteComplete(); } } diff --git a/Server/World.cs b/Server/World.cs index db2150a28..44907116a 100644 --- a/Server/World.cs +++ b/Server/World.cs @@ -45,7 +45,9 @@ namespace Server { private static bool m_Loading; private static bool m_Loaded; + private static bool m_Saving; + private static ManualResetEvent m_DiskWriteHandle = new ManualResetEvent(true); private static Queue _addQueue, _deleteQueue; @@ -64,6 +66,19 @@ namespace Server { public readonly static string GuildIndexPath = Path.Combine( "Saves/Guilds/", "Guilds.idx" ); public readonly static string GuildDataPath = Path.Combine( "Saves/Guilds/", "Guilds.bin" ); + public static void NotifyDiskWriteComplete() + { + if( m_DiskWriteHandle.Set()) + { + Console.WriteLine("Closing Save Files..."); + } + } + + public static void WaitForWriteCompletion() + { + m_DiskWriteHandle.WaitOne(); + } + public static Dictionary Mobiles { get { return m_Mobiles; } } @@ -758,19 +773,24 @@ namespace Server { internal static int m_Saves; public static void Save() { - ++m_Saves; - Save( true ); + Save( true, false ); } - public static void Save( bool message ) { - if ( m_Saving || AsyncWriter.ThreadCount > 0 ) + public static void Save( bool message, bool permitBackgroundWrite ) { + if ( m_Saving ) return; + ++m_Saves; + NetState.FlushAll(); NetState.Pause(); + World.WaitForWriteCompletion();//Blocks Save until current disk flush is done. + m_Saving = true; + m_DiskWriteHandle.Reset(); + if ( message ) Broadcast( 0x35, true, "The world is saving, please wait." ); @@ -790,7 +810,7 @@ namespace Server { /*using ( SaveMetrics metrics = new SaveMetrics() ) {*/ - strategy.Save( null ); + strategy.Save( null, permitBackgroundWrite ); /*}*/ try { @@ -803,11 +823,14 @@ namespace Server { m_Saving = false; + if (!permitBackgroundWrite) + World.NotifyDiskWriteComplete(); //Sets the DiskWriteHandle. If we allow background writes, we leave this upto the individual save strategies. + ProcessSafetyQueues(); strategy.ProcessDecay(); - Console.WriteLine( "done in {0:F2} seconds.", watch.Elapsed.TotalSeconds ); + Console.WriteLine( "Save done in {0:F2} seconds.", watch.Elapsed.TotalSeconds ); if ( message ) Broadcast( 0x35, true, "World save complete. The entire process took {0:F1} seconds.", watch.Elapsed.TotalSeconds );