fix: world save failure handling — no partial snapshots, no worker crash, staged publication (#2612)

Three pre-existing world-save failure-handling bugs, found while reviewing the delta-saves engine (#2610), split out so they land and get adopted on their own before the larger change. #2610 will be rebased on top once this merges. Tracked in project Delta Saves (#7).

## 1. A snapshot missing a segment was published

`GenericEntityPersistence.WriteSnapshot` logged a segment (or self-payload) write error and carried on. The save was published without those records, and every entity in the failed segment was deleted at the next load. The error now propagates: `WriteFiles` never moves a partial snapshot over `Saves/`, the previous save stays authoritative, and staff are told.

## 2. A serializer exception killed the process, or published a partial freeze

An entity whose `Serialize` threw on a worker thread was an unhandled exception on that thread, which terminates the process. On the inline (main-thread) drain it was caught, but the snapshot was still queued for writing with whatever the workers had produced. Workers now record the first exception and keep draining so the queue empties and the wake/pause handshake completes; after every worker paused, `Snapshot` treats a recorded error (or a failure of the drain itself) as a failed save: nothing is written, the previous save stays, staff are told, and the world resumes. A `WorldSave` handler throwing is logged but does not invalidate the serialized snapshot.

## 3. Publication was not transactional

Publishing moved the previous `Saves/` away (`AutoArchive` in `WorldSavePostSnapshot`) and only then moved the new files in. A subscriber throwing after the archive, or the final move failing, left nothing at `Saves/` until the next save; a crash in that window lost the newest save at the next boot.

The snapshot now moves to `Saves.next` as soon as its files are complete; only then do subscribers archive the previous save (same event, same `OldSavePath`, so `AutoArchive` and shard subscribers are unchanged), and the staged directory is renamed into place, which is atomic on one volume (file-by-file move across volumes). A staged directory is by construction a complete save newer than `Saves/`, so an interrupted publish is finished at the next boot (before load) or before the next save: whatever sits at `Saves/` is set aside as `Saves.previous-<timestamp>` (never deleted) and the staged save is published, with a warning naming the directory to archive or delete by hand.

## Tests

Server.Tests 860 / UOContent.Tests green. New: a throwing serializer is recorded on the worker and the next drain starts clean; a segment that cannot be indexed fails `WriteSnapshot` instead of dropping records; staged-save recovery with and without an existing `Saves/`, and as a no-op.

Projects/Server/World/World.cs

@@ -122,6 +122,8 @@ public static class World
         UseMultiThreadedSaves = ServerConfiguration.GetOrUpdateSetting("world.useMultithreadedSaves", true);
     }
 
+    internal static void SetSavePathForTest(string path) => SavePath = path;
+
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public static void WaitForWriteCompletion() => _diskWriteHandle.WaitOne();
 
@@ -195,6 +197,8 @@ public static class World
         logger.Information("Loading world");
         var watch = Stopwatch.StartNew();
 
+        RecoverStagedSave();
+
         Persistence.Load(SavePath);
         EventSink.InvokeWorldLoad();
 
@@ -271,6 +275,8 @@ public static class World
     {
         try
         {
+            RecoverStagedSave();
+
             // Allocate the heaps for the GC
             foreach (var worker in _threadWorkers)
             {
@@ -322,22 +328,49 @@ public static class World
             }
 
             Persistence.SerializeAll();
-            PauseSerializationThreads();
-            LogWorkerBalance();
-
-            EventSink.InvokeWorldSave();
         }
         catch (Exception ex)
         {
             exception = ex;
         }
 
-        WorldState = WorldState.WritingSave;
-        ThreadPool.QueueUserWorkItem(WriteFiles, snapshotPath);
+        // Always join the workers; any serializer exception fails the save.
+        try
+        {
+            PauseSerializationThreads();
+        }
+        catch (Exception ex)
+        {
+            exception ??= ex;
+        }
+
+        for (var i = 0; i < _threadWorkers.Length; i++)
+        {
+            exception ??= _threadWorkers[i].Error;
+        }
+
+        if (exception == null)
+        {
+            LogWorkerBalance();
+
+            try
+            {
+                EventSink.InvokeWorldSave();
+            }
+            catch (Exception ex)
+            {
+                logger.Error(ex, "A WorldSave handler failed");
+                Persistence.TraceException(ex);
+            }
+        }
+
         watch.Stop();
 
         if (exception == null)
         {
+            WorldState = WorldState.WritingSave;
+            ThreadPool.QueueUserWorkItem(WriteFiles, snapshotPath);
+
             var duration = watch.Elapsed.TotalSeconds;
             logger.Information("Saving world {Status} ({Duration:F2} seconds)", "done", duration);
 
@@ -349,6 +382,9 @@ public static class World
             Persistence.TraceException(exception);
 
             BroadcastStaff(0x35, true, "World save failed! Check the logs!");
+
+            _diskWriteHandle.Set();
+            FinishWorldSave();
         }
     }
 
@@ -361,17 +397,7 @@ public static class World
             logger.Information("Writing world save snapshot");
 
             Persistence.WriteSnapshotAll(snapshotPath);
-
-            try
-            {
-                EventSink.InvokeWorldSavePostSnapshot(SavePath, snapshotPath);
-                PathUtility.MoveDirectoryContents(snapshotPath, SavePath);
-                Directory.SetLastWriteTimeUtc(SavePath, Core.Now);
-            }
-            catch (Exception ex)
-            {
-                Persistence.TraceException(ex);
-            }
+            PublishSnapshot(snapshotPath);
 
             watch.Stop();
             logger.Information("Writing world save snapshot {Status} ({Duration:F2} seconds)", "done", watch.Elapsed.TotalSeconds);
@@ -388,6 +414,90 @@ public static class World
         Core.LoopContext.Post(FinishWorldSave);
     }
 
+    /// <summary>
+    /// A complete snapshot is staged here before the previous save is touched; a staged
+    /// directory is always a complete save newer than <see cref="SavePath" />.
+    /// </summary>
+    internal static string StagedSavePath => SavePath + ".next";
+
+    // Stage, let subscribers archive the previous save, then rename the staged save into place.
+    private static void PublishSnapshot(string snapshotPath)
+    {
+        var staging = StagedSavePath;
+
+        if (Directory.Exists(staging))
+        {
+            SetAside(staging, "unpublished");
+        }
+
+        MoveDirectory(snapshotPath, staging);
+        PublishStagedSave(archive: true);
+    }
+
+    private static void PublishStagedSave(bool archive)
+    {
+        var staging = StagedSavePath;
+
+        if (archive)
+        {
+            EventSink.InvokeWorldSavePostSnapshot(SavePath, staging);
+        }
+
+        if (Directory.Exists(SavePath))
+        {
+            SetAside(SavePath, "previous");
+        }
+
+        MoveDirectory(staging, SavePath);
+        Directory.SetLastWriteTimeUtc(SavePath, Core.Now);
+    }
+
+    /// <summary>
+    /// Finishes an interrupted publish. Runs at boot (before load) and before every save;
+    /// whatever is at Saves/ is set aside, never deleted.
+    /// </summary>
+    internal static void RecoverStagedSave()
+    {
+        var staging = StagedSavePath;
+
+        if (!Directory.Exists(staging))
+        {
+            return;
+        }
+
+        logger.Warning(
+            "A complete world save was staged at {Staging} but never published; publishing it now.",
+            staging
+        );
+
+        PublishStagedSave(archive: false);
+    }
+
+    private static void SetAside(string path, string reason)
+    {
+        var aside = $"{path}.{reason}-{Core.Now:yyyy-MM-dd-HH-mm-ss-fff}";
+        MoveDirectory(path, aside);
+        logger.Warning("Set aside {Path} as {Aside}; delete or archive it by hand.", path, aside);
+    }
+
+    // Atomic rename on one volume, file-by-file move otherwise.
+    private static void MoveDirectory(string source, string destination)
+    {
+        try
+        {
+            Directory.Move(source, destination);
+        }
+        catch (IOException)
+        {
+            if (Directory.Exists(destination))
+            {
+                throw;
+            }
+
+            PathUtility.MoveDirectoryContents(source, destination);
+        }
+    }
+
     private static void FinishWorldSave()
     {
         WorldState = WorldState.Running;