Fix for RunUO Crash if Performance tab on Admin gump is open within first 100seconds of server running.

Completion of the DynamicSaveStrategy.  Background saves are now safe; triggered only by the autoSave or command [BackgroundSave ([BGSave)
This commit is contained in:
asayre 2010-12-20 03:34:25 +00:00
parent 0afa707b24
commit 67af95cf0b
12 changed files with 169 additions and 84 deletions

View file

@ -48,6 +48,9 @@ namespace Server.Commands
Register( "Help", AccessLevel.Player, new CommandEventHandler( Help_OnCommand ) ); Register( "Help", AccessLevel.Player, new CommandEventHandler( Help_OnCommand ) );
Register( "Save", AccessLevel.Administrator, new CommandEventHandler( Save_OnCommand ) ); Register( "Save", AccessLevel.Administrator, new CommandEventHandler( Save_OnCommand ) );
Register( "BackgroundSave", AccessLevel.Administrator, new CommandEventHandler( BackgroundSave_OnCommand ) );
Register( "BGSave", AccessLevel.Administrator, new CommandEventHandler( BackgroundSave_OnCommand ) );
Register( "SaveBG", AccessLevel.Administrator, new CommandEventHandler( BackgroundSave_OnCommand ) );
Register( "Move", AccessLevel.GameMaster, new CommandEventHandler( Move_OnCommand ) ); Register( "Move", AccessLevel.GameMaster, new CommandEventHandler( Move_OnCommand ) );
Register( "Client", AccessLevel.Counselor, new CommandEventHandler( Client_OnCommand ) ); Register( "Client", AccessLevel.Counselor, new CommandEventHandler( Client_OnCommand ) );
@ -639,6 +642,14 @@ namespace Server.Commands
Misc.AutoSave.Save(); Misc.AutoSave.Save();
} }
[Usage("BackgroundSave")]
[Aliases("BGSave", "SaveBG")]
[Description("Saves the world, writing to the disk in the background")]
private static void BackgroundSave_OnCommand(CommandEventArgs e)
{
Misc.AutoSave.Save( true );
}
private static bool FixMap( ref Map map, ref Point3D loc, Item item ) private static bool FixMap( ref Map map, ref Point3D loc, Item item )
{ {
if ( map == null || map == Map.Internal ) if ( map == null || map == Map.Internal )

View file

@ -52,7 +52,7 @@ namespace Server.Misc
if ( m_Warning == TimeSpan.Zero ) if ( m_Warning == TimeSpan.Zero )
{ {
Save(); Save( true );
} }
else else
{ {
@ -72,14 +72,21 @@ namespace Server.Misc
} }
public static void Save() public static void Save()
{
AutoSave.Save( false );
}
public static void Save( bool permitBackgroundWrite )
{ {
if ( AutoRestart.Restarting ) if ( AutoRestart.Restarting )
return; return;
try{ Backup(); } World.WaitForWriteCompletion();
catch (Exception e) { Console.WriteLine("WARNING: Automatic backup FAILED: {0}", e); }
World.Save(); try{ Backup(); }
catch ( Exception e ) { Console.WriteLine("WARNING: Automatic backup FAILED: {0}", e); }
World.Save( true, permitBackgroundWrite );
} }
private static string[] m_Backups = new string[] private static string[] m_Backups = new string[]

View file

@ -25,11 +25,14 @@ namespace Server.Misc
public static void CrashGuard_OnCrash( CrashedEventArgs e ) public static void CrashGuard_OnCrash( CrashedEventArgs e )
{ {
if ( GenerateReport )
GenerateCrashReport( e );
World.WaitForWriteCompletion();
if ( SaveBackup ) if ( SaveBackup )
Backup(); Backup();
if ( GenerateReport )
GenerateCrashReport( e );
/*if ( Core.Service ) /*if ( Core.Service )
e.Close = true; e.Close = true;

View file

@ -302,7 +302,7 @@ namespace Server
if( World.Saving || ( m_Service && type == ConsoleEventType.CTRL_LOGOFF_EVENT ) ) if( World.Saving || ( m_Service && type == ConsoleEventType.CTRL_LOGOFF_EVENT ) )
return true; return true;
Kill(); Kill(); //Kill -> HandleClosed will hadnle waiting for the completion of flushign to disk
return true; return true;
} }
@ -315,7 +315,7 @@ namespace Server
private static bool m_Closing; private static bool m_Closing;
public static bool Closing { get { return m_Closing; } } public static bool Closing { get { return m_Closing; } }
private static long m_CycleIndex; private static long m_CycleIndex = 1;
private static float[] m_CyclesPerSecond = new float[100]; private static float[] m_CyclesPerSecond = new float[100];
public static float CyclesPerSecond public static float CyclesPerSecond
@ -364,6 +364,8 @@ namespace Server
Console.Write( "Exiting..." ); Console.Write( "Exiting..." );
World.WaitForWriteCompletion();
if( !m_Crashed ) if( !m_Crashed )
EventSink.InvokeShutdown( new ShutdownEventArgs() ); EventSink.InvokeShutdown( new ShutdownEventArgs() );

View file

@ -37,18 +37,24 @@ namespace Server {
public DualSaveStrategy() { public DualSaveStrategy() {
} }
public override void Save( SaveMetrics metrics ) { public override void Save( SaveMetrics metrics, bool permitBackgroundWrite )
{
this.PermitBackgroundWrite = permitBackgroundWrite;
Thread saveThread = new Thread( delegate() { Thread saveThread = new Thread( delegate() {
SaveItems( metrics ); SaveItems(metrics);
} ); } );
saveThread.Name = "Item Save Subset"; saveThread.Name = "Item Save Subset";
saveThread.Start(); saveThread.Start();
SaveMobiles( metrics ); SaveMobiles(metrics);
SaveGuilds( metrics ); SaveGuilds(metrics);
saveThread.Join(); saveThread.Join();
if (UseSequentialWriters)
World.NotifyDiskWriteComplete();
} }
} }
} }

View file

@ -59,7 +59,7 @@ namespace Server
_guildThreadWriters = new BlockingCollection<QueuedMemoryWriter>(); _guildThreadWriters = new BlockingCollection<QueuedMemoryWriter>();
} }
public override void Save(SaveMetrics metrics) public override void Save(SaveMetrics metrics, bool permitBackgroundWrite)
{ {
this._metrics = metrics; this._metrics = metrics;
@ -73,13 +73,21 @@ namespace Server
SaveTypeDatabases(); SaveTypeDatabases();
if (permitBackgroundWrite)
{
//This option makes it finish the writing to disk in the background, continuing even after Save() returns.
Task.Factory.ContinueWhenAll(saveTasks, _ =>
{
CloseFiles();
Task.WaitAll(saveTasks); //Waits for the completion of all of the tasks(committing to disk) World.NotifyDiskWriteComplete();
CloseFiles(); });
}
//This option makes it finish the writing to disk in the background, continuing even after Save() returns. VERY DANGEROUS. else
//TODO: Make this less dangerous, by notifying rest of the server that this can and does happen {
//Task.Factory.ContinueWhenAll(saveTasks, _ => CloseFiles()); Task.WaitAll(saveTasks); //Waits for the completion of all of the tasks(committing to disk)
CloseFiles();
}
} }
private Task StartCommitTask(BlockingCollection<QueuedMemoryWriter> threadWriter, SequentialFileWriter data, SequentialFileWriter index) private Task StartCommitTask(BlockingCollection<QueuedMemoryWriter> threadWriter, SequentialFileWriter data, SequentialFileWriter index)
@ -264,8 +272,6 @@ namespace Server
_guildData.Close(); _guildData.Close();
_guildIndex.Close(); _guildIndex.Close();
Console.WriteLine("Closing files");
} }
private void WriteCount(SequentialFileWriter indexFile, int count) private void WriteCount(SequentialFileWriter indexFile, int count)

View file

@ -60,7 +60,8 @@ namespace Server {
private bool finished; private bool finished;
public override void Save( SaveMetrics metrics ) { public override void Save(SaveMetrics metrics, bool permitBackgroundWrite)
{
this.metrics = metrics; this.metrics = metrics;
OpenFiles(); OpenFiles();
@ -163,6 +164,8 @@ namespace Server {
guildData.Close(); guildData.Close();
guildIndex.Close(); guildIndex.Close();
World.NotifyDiskWriteComplete();
} }
private void OnSerialized( ConsumableEntry entry ) { private void OnSerialized( ConsumableEntry entry ) {

View file

@ -25,7 +25,6 @@ using System.Text;
namespace Server namespace Server
{ {
public sealed class QueuedMemoryWriter : BinaryFileWriter public sealed class QueuedMemoryWriter : BinaryFileWriter
{ {
private struct IndexInfo private struct IndexInfo
@ -36,11 +35,11 @@ namespace Server
} }
private MemoryStream _memStream; private MemoryStream _memStream;
private Queue<IndexInfo> _indexQueue = new Queue<IndexInfo>(); //TODO: Pick a more optimal starting size private List<IndexInfo> _orderedIndexInfo = new List<IndexInfo>();
protected override int BufferSize protected override int BufferSize
{ {
get { return 4096; } get { return 512; }
} }
public QueuedMemoryWriter() public QueuedMemoryWriter()
@ -58,66 +57,70 @@ namespace Server
info.typeCode = serializable.TypeReference; //For guilds, this will automagically be zero. info.typeCode = serializable.TypeReference; //For guilds, this will automagically be zero.
info.serial = serializable.SerialIdentity; info.serial = serializable.SerialIdentity;
_indexQueue.Enqueue(info); _orderedIndexInfo.Add(info);
} }
public void CommitTo(SequentialFileWriter dataFile, SequentialFileWriter indexFile) public void CommitTo(SequentialFileWriter dataFile, SequentialFileWriter indexFile)
{ {
this.Flush(); this.Flush();
byte[] memBuffer = _memStream.GetBuffer();
int memLength = (int)_memStream.Position; int memLength = (int)_memStream.Position;
long actualPosition = dataFile.Position; if (memLength > 0)
dataFile.Write(memBuffer, 0, memLength); //The buffer contains the data from many items.
//Console.WriteLine("Writing {0} bytes starting at {1}", memLength, actualPosition);
byte[] indexBuffer = new byte[20];
//int indexWritten = _indexQueue.Count * indexBuffer.Length;
//int totalWritten = memLength + indexWritten
while (_indexQueue.Count > 0)
{ {
IndexInfo info = _indexQueue.Dequeue(); byte[] memBuffer = _memStream.GetBuffer();
int typeCode = info.typeCode; long actualPosition = dataFile.Position;
int serial = info.serial;
int length = info.size; dataFile.Write(memBuffer, 0, memLength); //The buffer contains the data from many items.
//Console.WriteLine("Writing {0} bytes starting at {1}, with {2} things", memLength, actualPosition, _orderedIndexInfo.Count);
byte[] indexBuffer = new byte[20];
//int indexWritten = _orderedIndexInfo.Count * indexBuffer.Length;
//int totalWritten = memLength + indexWritten
for (int i = 0; i < _orderedIndexInfo.Count; i++)
{
IndexInfo info = _orderedIndexInfo[i];
int typeCode = info.typeCode;
int serial = info.serial;
int length = info.size;
indexBuffer[0] = (byte)(info.typeCode); indexBuffer[0] = (byte)(info.typeCode);
indexBuffer[1] = (byte)(info.typeCode >> 8); indexBuffer[1] = (byte)(info.typeCode >> 8);
indexBuffer[2] = (byte)(info.typeCode >> 16); indexBuffer[2] = (byte)(info.typeCode >> 16);
indexBuffer[3] = (byte)(info.typeCode >> 24); indexBuffer[3] = (byte)(info.typeCode >> 24);
indexBuffer[4] = (byte)(info.serial); indexBuffer[4] = (byte)(info.serial);
indexBuffer[5] = (byte)(info.serial >> 8); indexBuffer[5] = (byte)(info.serial >> 8);
indexBuffer[6] = (byte)(info.serial >> 16); indexBuffer[6] = (byte)(info.serial >> 16);
indexBuffer[7] = (byte)(info.serial >> 24); indexBuffer[7] = (byte)(info.serial >> 24);
indexBuffer[8] = (byte)(actualPosition); indexBuffer[8] = (byte)(actualPosition);
indexBuffer[9] = (byte)(actualPosition >> 8); indexBuffer[9] = (byte)(actualPosition >> 8);
indexBuffer[10] = (byte)(actualPosition >> 16); indexBuffer[10] = (byte)(actualPosition >> 16);
indexBuffer[11] = (byte)(actualPosition >> 24); indexBuffer[11] = (byte)(actualPosition >> 24);
indexBuffer[12] = (byte)(actualPosition >> 32); indexBuffer[12] = (byte)(actualPosition >> 32);
indexBuffer[13] = (byte)(actualPosition >> 40); indexBuffer[13] = (byte)(actualPosition >> 40);
indexBuffer[14] = (byte)(actualPosition >> 48); indexBuffer[14] = (byte)(actualPosition >> 48);
indexBuffer[15] = (byte)(actualPosition >> 56); indexBuffer[15] = (byte)(actualPosition >> 56);
indexBuffer[16] = (byte)(info.size); indexBuffer[16] = (byte)(info.size);
indexBuffer[17] = (byte)(info.size >> 8); indexBuffer[17] = (byte)(info.size >> 8);
indexBuffer[18] = (byte)(info.size >> 16); indexBuffer[18] = (byte)(info.size >> 16);
indexBuffer[19] = (byte)(info.size >> 24); indexBuffer[19] = (byte)(info.size >> 24);
indexFile.Write(indexBuffer, 0, indexBuffer.Length); indexFile.Write(indexBuffer, 0, indexBuffer.Length);
actualPosition += info.size; actualPosition += info.size;
}
} }
this.Close(); //We're donezo with this writer. this.Close(); //We're done with this writer.
} }
} }
} }

View file

@ -51,7 +51,7 @@ namespace Server
} }
public abstract string Name { get; } public abstract string Name { get; }
public abstract void Save(SaveMetrics metrics); public abstract void Save(SaveMetrics metrics, bool permitBackgroundWrite);
public abstract void ProcessDecay(); public abstract void ProcessDecay();
} }

View file

@ -35,25 +35,38 @@ namespace Server {
} }
private Queue<Item> _decayQueue; private Queue<Item> _decayQueue;
private bool _permitBackgroundWrite;
public StandardSaveStrategy() { public StandardSaveStrategy() {
_decayQueue = new Queue<Item>(); _decayQueue = new Queue<Item>();
} }
public override void Save( SaveMetrics metrics ) { protected bool PermitBackgroundWrite { get { return _permitBackgroundWrite; } set { _permitBackgroundWrite = value; } }
SaveMobiles( metrics );
SaveItems( metrics ); protected bool UseSequentialWriters { get { return (World.SaveType == World.SaveOption.Normal || !_permitBackgroundWrite); } }
SaveGuilds( metrics );
public override void Save(SaveMetrics metrics, bool permitBackgroundWrite)
{
_permitBackgroundWrite = permitBackgroundWrite;
SaveMobiles(metrics);
SaveItems(metrics);
SaveGuilds(metrics);
if (UseSequentialWriters)
World.NotifyDiskWriteComplete();
} }
protected void SaveMobiles( SaveMetrics metrics ) { protected void SaveMobiles(SaveMetrics metrics)
{
Dictionary<Serial, Mobile> mobiles = World.Mobiles; Dictionary<Serial, Mobile> mobiles = World.Mobiles;
GenericWriter idx; GenericWriter idx;
GenericWriter tdb; GenericWriter tdb;
GenericWriter bin; GenericWriter bin;
if ( World.SaveType == World.SaveOption.Normal ) { if (UseSequentialWriters)
{
idx = new BinaryFileWriter( World.MobileIndexPath, false ); idx = new BinaryFileWriter( World.MobileIndexPath, false );
tdb = new BinaryFileWriter( World.MobileTypesPath, false ); tdb = new BinaryFileWriter( World.MobileTypesPath, false );
bin = new BinaryFileWriter( World.MobileDataPath, true ); bin = new BinaryFileWriter( World.MobileDataPath, true );
@ -92,7 +105,8 @@ namespace Server {
bin.Close(); bin.Close();
} }
protected void SaveItems( SaveMetrics metrics ) { protected void SaveItems(SaveMetrics metrics)
{
Dictionary<Serial, Item> items = World.Items; Dictionary<Serial, Item> items = World.Items;
List<Item> decaying = new List<Item>(); List<Item> decaying = new List<Item>();
@ -100,7 +114,8 @@ namespace Server {
GenericWriter tdb; GenericWriter tdb;
GenericWriter bin; GenericWriter bin;
if ( World.SaveType == World.SaveOption.Normal ) { if (UseSequentialWriters)
{
idx = new BinaryFileWriter( World.ItemIndexPath, false ); idx = new BinaryFileWriter( World.ItemIndexPath, false );
tdb = new BinaryFileWriter( World.ItemTypesPath, false ); tdb = new BinaryFileWriter( World.ItemTypesPath, false );
bin = new BinaryFileWriter( World.ItemDataPath, true ); bin = new BinaryFileWriter( World.ItemDataPath, true );
@ -142,11 +157,13 @@ namespace Server {
bin.Close(); bin.Close();
} }
protected void SaveGuilds( SaveMetrics metrics ) { protected void SaveGuilds(SaveMetrics metrics)
{
GenericWriter idx; GenericWriter idx;
GenericWriter bin; GenericWriter bin;
if ( World.SaveType == World.SaveOption.Normal ) { if (UseSequentialWriters)
{
idx = new BinaryFileWriter( World.GuildIndexPath, false ); idx = new BinaryFileWriter( World.GuildIndexPath, false );
bin = new BinaryFileWriter( World.GuildDataPath, true ); bin = new BinaryFileWriter( World.GuildDataPath, true );
} else { } else {

View file

@ -1217,7 +1217,11 @@ namespace Server
if( m_Owner.m_Closed ) if( m_Owner.m_Closed )
m_Owner.m_File.Close(); m_Owner.m_File.Close();
AsyncWriter.m_ThreadCount--; AsyncWriter.m_ThreadCount--;
if (AsyncWriter.m_ThreadCount <= 0)
World.NotifyDiskWriteComplete();
} }
} }

View file

@ -45,7 +45,9 @@ namespace Server {
private static bool m_Loading; private static bool m_Loading;
private static bool m_Loaded; private static bool m_Loaded;
private static bool m_Saving; private static bool m_Saving;
private static ManualResetEvent m_DiskWriteHandle = new ManualResetEvent(true);
private static Queue<IEntity> _addQueue, _deleteQueue; private static Queue<IEntity> _addQueue, _deleteQueue;
@ -64,6 +66,19 @@ namespace Server {
public readonly static string GuildIndexPath = Path.Combine( "Saves/Guilds/", "Guilds.idx" ); public readonly static string GuildIndexPath = Path.Combine( "Saves/Guilds/", "Guilds.idx" );
public readonly static string GuildDataPath = Path.Combine( "Saves/Guilds/", "Guilds.bin" ); public readonly static string GuildDataPath = Path.Combine( "Saves/Guilds/", "Guilds.bin" );
public static void NotifyDiskWriteComplete()
{
if( m_DiskWriteHandle.Set())
{
Console.WriteLine("Closing Save Files...");
}
}
public static void WaitForWriteCompletion()
{
m_DiskWriteHandle.WaitOne();
}
public static Dictionary<Serial, Mobile> Mobiles { public static Dictionary<Serial, Mobile> Mobiles {
get { return m_Mobiles; } get { return m_Mobiles; }
} }
@ -758,19 +773,24 @@ namespace Server {
internal static int m_Saves; internal static int m_Saves;
public static void Save() { public static void Save() {
++m_Saves; Save( true, false );
Save( true );
} }
public static void Save( bool message ) { public static void Save( bool message, bool permitBackgroundWrite ) {
if ( m_Saving || AsyncWriter.ThreadCount > 0 ) if ( m_Saving )
return; return;
++m_Saves;
NetState.FlushAll(); NetState.FlushAll();
NetState.Pause(); NetState.Pause();
World.WaitForWriteCompletion();//Blocks Save until current disk flush is done.
m_Saving = true; m_Saving = true;
m_DiskWriteHandle.Reset();
if ( message ) if ( message )
Broadcast( 0x35, true, "The world is saving, please wait." ); Broadcast( 0x35, true, "The world is saving, please wait." );
@ -790,7 +810,7 @@ namespace Server {
/*using ( SaveMetrics metrics = new SaveMetrics() ) {*/ /*using ( SaveMetrics metrics = new SaveMetrics() ) {*/
strategy.Save( null ); strategy.Save( null, permitBackgroundWrite );
/*}*/ /*}*/
try { try {
@ -803,11 +823,14 @@ namespace Server {
m_Saving = false; m_Saving = false;
if (!permitBackgroundWrite)
World.NotifyDiskWriteComplete(); //Sets the DiskWriteHandle. If we allow background writes, we leave this upto the individual save strategies.
ProcessSafetyQueues(); ProcessSafetyQueues();
strategy.ProcessDecay(); strategy.ProcessDecay();
Console.WriteLine( "done in {0:F2} seconds.", watch.Elapsed.TotalSeconds ); Console.WriteLine( "Save done in {0:F2} seconds.", watch.Elapsed.TotalSeconds );
if ( message ) if ( message )
Broadcast( 0x35, true, "World save complete. The entire process took {0:F1} seconds.", watch.Elapsed.TotalSeconds ); Broadcast( 0x35, true, "World save complete. The entire process took {0:F1} seconds.", watch.Elapsed.TotalSeconds );