fix: world save failure handling — no partial snapshots, no worker crash, staged publication (#2612)

Three pre-existing world-save failure-handling bugs, found while reviewing the delta-saves engine (#2610), split out so they land and get adopted on their own before the larger change. #2610 will be rebased on top once this merges. Tracked in project Delta Saves (#7).

## 1. A snapshot missing a segment was published

`GenericEntityPersistence.WriteSnapshot` logged a segment (or self-payload) write error and carried on. The save was published without those records, and every entity in the failed segment was deleted at the next load. The error now propagates: `WriteFiles` never moves a partial snapshot over `Saves/`, the previous save stays authoritative, and staff are told.

## 2. A serializer exception killed the process, or published a partial freeze

An entity whose `Serialize` threw on a worker thread was an unhandled exception on that thread, which terminates the process. On the inline (main-thread) drain it was caught, but the snapshot was still queued for writing with whatever the workers had produced. Workers now record the first exception and keep draining so the queue empties and the wake/pause handshake completes; after every worker paused, `Snapshot` treats a recorded error (or a failure of the drain itself) as a failed save: nothing is written, the previous save stays, staff are told, and the world resumes. A `WorldSave` handler throwing is logged but does not invalidate the serialized snapshot.

## 3. Publication was not transactional

Publishing moved the previous `Saves/` away (`AutoArchive` in `WorldSavePostSnapshot`) and only then moved the new files in. A subscriber throwing after the archive, or the final move failing, left nothing at `Saves/` until the next save; a crash in that window lost the newest save at the next boot.

The snapshot now moves to `Saves.next` as soon as its files are complete; only then do subscribers archive the previous save (same event, same `OldSavePath`, so `AutoArchive` and shard subscribers are unchanged), and the staged directory is renamed into place, which is atomic on one volume (file-by-file move across volumes). A staged directory is by construction a complete save newer than `Saves/`, so an interrupted publish is finished at the next boot (before load) or before the next save: whatever sits at `Saves/` is set aside as `Saves.previous-<timestamp>` (never deleted) and the staged save is published, with a warning naming the directory to archive or delete by hand.

## Tests

Server.Tests 860 / UOContent.Tests green. New: a throwing serializer is recorded on the worker and the next drain starts clean; a segment that cannot be indexed fails `WriteSnapshot` instead of dropping records; staged-save recovery with and without an existing `Saves/`, and as a no-op.

Projects/Server.Tests/Tests/Serialization/GenericEntityPersistenceRoundTripTests.cs

@@ -20,7 +20,7 @@ internal class RoundTripEntity : ISerializable
     {
     }
 
-    public void Serialize(IGenericWriter writer)
+    public virtual void Serialize(IGenericWriter writer)
     {
         writer.Write(Value);
         writer.Write(Name);
@@ -33,6 +33,15 @@ internal class RoundTripEntity : ISerializable
     }
 }
 
+internal class ThrowingRoundTripEntity : RoundTripEntity
+{
+    public ThrowingRoundTripEntity(Serial serial) : base(serial)
+    {
+    }
+
+    public override void Serialize(IGenericWriter writer) => throw new InvalidOperationException("broken serializer");
+}
+
 [Collection("Sequential Server Tests")]
 public class GenericEntityPersistenceRoundTripTests
 {
@@ -161,4 +170,143 @@ public class GenericEntityPersistenceRoundTripTests
             Directory.Delete(dir, true);
         }
     }
+
+    /// <summary>
+    /// A serializer throwing on a worker used to be an unhandled exception on that thread.
+    /// The worker records it, finishes the drain so the handshake completes, and the loop
+    /// fails the save.
+    /// </summary>
+    [Fact]
+    public void SerializerException_IsRecordedOnTheWorker_AndTheDrainCompletes()
+    {
+        var source = new SerializationChunkSource();
+        var workers = new SerializationThreadWorker[2];
+        for (var i = 0; i < workers.Length; i++)
+        {
+            workers[i] = new SerializationThreadWorker(i, source);
+            workers[i].AllocateHeap();
+        }
+
+        var persistence = new RoundTripPersistence(2002);
+
+        try
+        {
+            for (var i = 1; i <= 100; i++)
+            {
+                var serial = (Serial)(uint)i;
+                persistence.EntitiesBySerial[serial] = i == 50
+                    ? new ThrowingRoundTripEntity(serial)
+                    : new RoundTripEntity(serial) { Value = i, Name = $"entity-{i}" };
+            }
+
+            foreach (var worker in workers)
+            {
+                worker.Wake();
+            }
+
+            source.SetOwner(persistence);
+            Assert.True(persistence.TrySnapshotEntries(out var slotCount));
+            source.PushSlotRanges(persistence, slotCount);
+            source.Flush();
+
+            foreach (var worker in workers)
+            {
+                worker.Sleep();
+            }
+
+            Exception error = null;
+            foreach (var worker in workers)
+            {
+                error ??= worker.Error;
+            }
+
+            Assert.IsType<InvalidOperationException>(error);
+            persistence.PostWorldSave();
+
+            // The next drain starts clean.
+            foreach (var worker in workers)
+            {
+                worker.Wake();
+            }
+
+            foreach (var worker in workers)
+            {
+                worker.Sleep();
+                Assert.Null(worker.Error);
+            }
+        }
+        finally
+        {
+            persistence.Unregister();
+
+            foreach (var worker in workers)
+            {
+                worker.Exit();
+            }
+        }
+    }
+
+    /// <summary>
+    /// A segment that cannot be written used to be logged and dropped, publishing a save
+    /// without those entities (the loader then deletes them). It now fails the save.
+    /// </summary>
+    [Fact]
+    public void WriteSnapshot_FailsTheSave_InsteadOfDroppingASegment()
+    {
+        var source = new SerializationChunkSource();
+        var workers = new SerializationThreadWorker[2];
+        for (var i = 0; i < workers.Length; i++)
+        {
+            workers[i] = new SerializationThreadWorker(i, source);
+            workers[i].AllocateHeap();
+        }
+
+        var previousWorkers = World._threadWorkers;
+        World._threadWorkers = workers;
+
+        var persistence = new RoundTripPersistence(2003);
+        var dir = Path.Combine(Path.GetTempPath(), $"muo-segmentfail-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(dir);
+
+        try
+        {
+            for (var i = 1; i <= 100; i++)
+            {
+                var serial = (Serial)(uint)i;
+                persistence.EntitiesBySerial[serial] = new RoundTripEntity(serial) { Value = i, Name = $"entity-{i}" };
+            }
+
+            // Deliberately not registered: the writer cannot index the type.
+
+            foreach (var worker in workers)
+            {
+                worker.Wake();
+            }
+
+            source.SetOwner(persistence);
+            Assert.True(persistence.TrySnapshotEntries(out var slotCount));
+            source.PushSlotRanges(persistence, slotCount);
+            source.Flush();
+
+            foreach (var worker in workers)
+            {
+                worker.Sleep();
+            }
+
+            Assert.Throws<InvalidOperationException>(() => persistence.WriteSnapshot(dir));
+            persistence.PostWorldSave();
+        }
+        finally
+        {
+            persistence.Unregister();
+
+            foreach (var worker in workers)
+            {
+                worker.Exit();
+            }
+
+            World._threadWorkers = previousWorkers;
+            Directory.Delete(dir, true);
+        }
+    }
 }

Projects/Server.Tests/Tests/Serialization/StagedSavePublishTests.cs

@@ -0,0 +1,83 @@
+using System;
+using System.IO;
+using Xunit;
+
+namespace Server.Tests;
+
+/// <summary>
+/// The publish protocol: a complete snapshot is staged next to Saves/ before the previous
+/// save is touched, and an interrupted publish is finished at the next boot or save.
+/// </summary>
+[Collection("Sequential Server Tests")]
+public class StagedSavePublishTests : IDisposable
+{
+    private readonly string _root = Path.Combine(Path.GetTempPath(), $"muo-staged-{Guid.NewGuid():N}");
+    private readonly string _previousSavePath;
+
+    public StagedSavePublishTests()
+    {
+        Directory.CreateDirectory(_root);
+        _previousSavePath = World.SavePath;
+        World.SetSavePathForTest(Path.Combine(_root, "Saves"));
+    }
+
+    public void Dispose()
+    {
+        World.SetSavePathForTest(_previousSavePath);
+
+        try
+        {
+            Directory.Delete(_root, true);
+        }
+        catch
+        {
+            // best effort
+        }
+    }
+
+    private static void WriteMarker(string dir, string name)
+    {
+        Directory.CreateDirectory(dir);
+        File.WriteAllText(Path.Combine(dir, "marker.txt"), name);
+    }
+
+    private static string ReadMarker(string dir) => File.ReadAllText(Path.Combine(dir, "marker.txt"));
+
+    [Fact]
+    public void NothingStaged_RecoveryIsANoOp()
+    {
+        WriteMarker(World.SavePath, "current");
+
+        World.RecoverStagedSave();
+
+        Assert.Equal("current", ReadMarker(World.SavePath));
+        Assert.Single(Directory.GetDirectories(_root));
+    }
+
+    [Fact]
+    public void StagedSave_ReplacesSaves_AndKeepsThePreviousOne()
+    {
+        WriteMarker(World.SavePath, "old");
+        WriteMarker(World.StagedSavePath, "new");
+
+        World.RecoverStagedSave();
+
+        Assert.Equal("new", ReadMarker(World.SavePath));
+        Assert.False(Directory.Exists(World.StagedSavePath));
+
+        var aside = Array.FindAll(Directory.GetDirectories(_root), d => d.Contains(".previous-"));
+        Assert.Single(aside);
+        Assert.Equal("old", ReadMarker(aside[0]));
+    }
+
+    [Fact]
+    public void StagedSave_WithNoSaves_IsPublished()
+    {
+        WriteMarker(World.StagedSavePath, "new");
+
+        World.RecoverStagedSave();
+
+        Assert.Equal("new", ReadMarker(World.SavePath));
+        Assert.Single(Directory.GetDirectories(_root));
+    }
+}