diff --git a/Projects/Server/Diagnostics/EventLoopProfiler.cs b/Projects/Server/Diagnostics/EventLoopProfiler.cs index a3fd157f5..615a84afd 100644 --- a/Projects/Server/Diagnostics/EventLoopProfiler.cs +++ b/Projects/Server/Diagnostics/EventLoopProfiler.cs @@ -26,6 +26,7 @@ public enum LoopPhase TimerSlice, NetworkSlice, LoopTasks, + WorldSnapshot, } /// @@ -42,7 +43,7 @@ public enum LoopPhase /// public static class EventLoopProfiler { - public const int PhaseCount = 5; + public const int PhaseCount = 6; private const int RingSize = 900; private const long SampleIntervalMs = 1000; diff --git a/Projects/Server/Main.cs b/Projects/Server/Main.cs index eb95e6d4e..620d67c28 100644 --- a/Projects/Server/Main.cs +++ b/Projects/Server/Main.cs @@ -57,6 +57,12 @@ public static class Core /// public static int EventLoopIdleWaitMs => _eventLoopIdleWaitMs; + /// + /// True when idle sleeping was disabled at startup because the host cannot honor short + /// waits, overriding whatever server.eventLoopIdleWaitMs was configured to. + /// + public static bool IdleSleepUnsupported { get; private set; } + /// /// Whether idle sleeping is currently suspended because the host returned waits late. /// @@ -129,18 +135,13 @@ public static class Core return; } - // Already suspended: extend rather than counting a fresh backoff episode. - if (_tickCount - _idleSleepSuspendedUntil < 0) - { - _idleSleepSuspendedUntil = _tickCount + _currentBackoffMs; - return; - } - - // A long clean streak resets the escalation. Gated on the count rather than a + // A long clean streak resets the escalation, re-arming the ceiling Error so a host that + // recovers and later degrades again gets re-reported. Gated on the count rather than a // "_lastBackoffAt > 0" sentinel because tick counts are not guaranteed positive. if (_consecutiveBackoffs > 0 && _tickCount - _lastBackoffAt > BackoffResetAfterCleanMs) { _consecutiveBackoffs = 0; + _loggedBackoffCeiling = false; } _currentBackoffMs = Math.Min(BackoffBaseMs << Math.Min(_consecutiveBackoffs, BackoffMaxShift), BackoffMaxMs); @@ -553,11 +554,29 @@ public static class Core ServerConfiguration.Load(); // 0 disables idle sleeping entirely (full-core spin, zero scheduling overhead). - _eventLoopIdleWaitMs = ServerConfiguration.GetSetting("server.eventLoopIdleWaitMs", 2); + var idleWaitMs = ServerConfiguration.GetSetting("server.eventLoopIdleWaitMs", 2); + if (idleWaitMs < 0) + { + logger.Warning( + "server.eventLoopIdleWaitMs {Value} is negative; using 0 (idle sleeping disabled)", + idleWaitMs + ); + } + + _eventLoopIdleWaitMs = Math.Max(0, idleWaitMs); // 16ms-budget misses per second before idle sleeping backs off. Raise to tolerate a // jittery host; set very high to disable the backoff. - _lateWakeThreshold = ServerConfiguration.GetSetting("server.lateWakeThreshold", 1); + var lateWakeThreshold = ServerConfiguration.GetSetting("server.lateWakeThreshold", 1); + if (lateWakeThreshold < 0) + { + logger.Warning( + "server.lateWakeThreshold {Value} is negative; using 0", + lateWakeThreshold + ); + } + + _lateWakeThreshold = Math.Max(0, lateWakeThreshold); var assemblyPath = Path.Join(BaseDirectory, AssembliesConfiguration); @@ -621,6 +640,7 @@ public static class Core "resolution failed). Idle sleeping is disabled. The loop will spin instead, using a full core." ); + IdleSleepUnsupported = true; _eventLoopIdleWaitMs = 0; } @@ -672,8 +692,10 @@ public static class Core if (_performSnapshot) { + EventLoopProfiler.PhaseStart(LoopPhase.WorldSnapshot); // Return value is the offset that can be used to fix timers that should drift World.Snapshot(_snapshotPath); + EventLoopProfiler.PhaseEnd(LoopPhase.WorldSnapshot); _performSnapshot = false; } diff --git a/Projects/Server/Network/NetState/NetState.Network.cs b/Projects/Server/Network/NetState/NetState.Network.cs index 9cb7b85fb..f69f92076 100644 --- a/Projects/Server/Network/NetState/NetState.Network.cs +++ b/Projects/Server/Network/NetState/NetState.Network.cs @@ -87,7 +87,7 @@ public partial class NetState /// internal static bool IsIdle => _throttled.Count == 0 && _throttledPending.Count == 0 && - _flushPending.Count == 0 && _disposed.Count == 0; + _flushPending.Count == 0 && _pendingDisconnects.Count == 0 && _disposed.Count == 0; /// /// Gets the listening addresses that the server is bound to. diff --git a/Projects/UOContent/Gumps/AdminGump.cs b/Projects/UOContent/Gumps/AdminGump.cs index 130d031fc..a4fbe0f18 100644 --- a/Projects/UOContent/Gumps/AdminGump.cs +++ b/Projects/UOContent/Gumps/AdminGump.cs @@ -227,7 +227,8 @@ namespace Server.Gumps } case AdminGumpPage.Information_Perf: { - var loopStatus = Core.EventLoopIdleWaitMs == 0 ? "Spinning (configured)" : + var loopStatus = Core.IdleSleepUnsupported ? "Spinning - host cannot honor short waits" : + Core.EventLoopIdleWaitMs == 0 ? "Spinning (configured)" : Core.IdleSleepSuspended ? "Sleep suspended - host returning waits late" : "Healthy"; AddLabel(20, 130, LabelHue, "Event Loop:"); diff --git a/dev-docs/debugging-event-loop.md b/dev-docs/debugging-event-loop.md index 5c5ea3b63..8b5fdcf36 100644 --- a/dev-docs/debugging-event-loop.md +++ b/dev-docs/debugging-event-loop.md @@ -8,7 +8,8 @@ funnel in order; most incidents resolve before the last step. Do not start with Every second of the main thread's wall time goes to exactly one of four places: 1. **Work** — the loop's phases: mobile deltas, item deltas, timer callbacks (`Timer.Slice`), - network processing (`NetState.Slice`), posted tasks (`LoopContext`). + network processing (`NetState.Slice`), posted tasks (`LoopContext`), world snapshots + (`WorldSnapshot` — the on-loop portion of a save). 2. **Sleep** — idle blocking in `NetState.WaitForCompletion`, bounded by the next timer tick and `server.eventLoopIdleWaitMs`. 3. **GC pauses** — land inside whichever phase (or sleep) was running. @@ -27,7 +28,7 @@ No build changes needed. Three signals exist, all actionable: |---|---|---| | Startup error: *host cannot honour short waits* | No high-resolution timer and `timeBeginPeriod` failed. Very old or unusual Windows. | Nothing is wrong with the server; it spins and uses a full core. Upgrade the OS or accept the core. | | Warning: *host returned a Nms idle wait late* + sleeping suspended | The OS did not reschedule the process promptly after a 1–2ms wait. Shared/burstable vCPU signature. | Move to dedicated CPU, or set `server.eventLoopIdleWaitMs=0` to spin permanently. This is a **host** problem — no amount of server-side change fixes it. | -| Admin gump → Performance → *Event Loop* | `Healthy` / `Sleep suspended (host)` / `Spinning (configured)` | Same as above. | +| Admin gump → Performance → *Event Loop* | `Healthy` / `Sleep suspended (host)` / `Spinning (configured)` / `Spinning - host cannot honor short waits` | Same as above; the last verdict is the startup error's state, not a config choice. | If none of these fired and the shard still feels laggy, the cause is work, GC, or something a boot-time signal cannot see. Continue.