feat(admin): replace dead ThreadPool counters with a network readout (#2663)
## Summary
The admin gump's **Performance** page still rendered `ThreadPool` worker / IOCP quotas — a RunUO leftover from `Socket.BeginReceive` days. Both figures are dead: IORingGroup drives RIO / io_uring / epoll / kqueue synchronously on the game loop, so the IOCP figure reads 0% forever, and the real off-loop work (serialization workers, password checks, advanced search, console input) runs on dedicated threads the worker figure never sees.
This replaces the block with a read-only network readout.
## Engine
- `NetState.GetNetworkStats()` — new public accessor returning a `NetworkStats` snapshot (ModernUO-owned record types in `Network/NetworkStats.cs`, so UOContent takes no dependency on transport types):
- live: connected / max sockets, authenticated count, per-tier growth usage, the three loop queues (throttled, flush pending, pending disconnects)
- configured: base / pre-auth / max buffer sizes, growth budget, memory ceiling, last-read available memory
- `LastSweep`: everything the once-a-minute maintenance sweep saw, including the refusal counters it resets
- The sweep already kept four of its results (`_lastTier*`, `_lastBaseCapacityBytes`) for change detection; those fold into one `SendBufferSweep` snapshot. Sweep logic and logging are unchanged.
- The accessor never calls `Maintain()`, so opening the page cannot rotate the retention window.
- Tick age is compared by subtraction; "has a sweep run" is a `bool`, not a zero-tick sentinel.
## Gump
```
Event Loop: Healthy
Connections: 123 / 4096 (117 authenticated)
Loop Queues: throttled 0, flush 3, closing 0
Send Buffers: [scrollable]
Base: 64.0 KB recv, 256.0 KB send; pre-auth 4.0 KB / 4.0 KB
Base pools: 10.2 MB
Growth tiers (live):
512.0 KB: 3 / 32 in use, floor 5
1.0 MB: 0 / 0 in use, floor 0
2.0 MB: 0 / 0 in use, floor 0
Budget: 16.0 MB of 256.0 MB, max 2.0 MB per socket
Ceiling: 80% of 7.8 GB available
Last sweep 42s ago: released 0 tier, 0 base
Refused: budget 0, at max 0, ceiling 0
```
This commit is contained in:
parent
24bcfee554
commit
29a3ab1bd4
4 changed files with 280 additions and 30 deletions
80
Projects/Server.Tests/Tests/Network/NetworkStatsTests.cs
Normal file
80
Projects/Server.Tests/Tests/Network/NetworkStatsTests.cs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
using System.Net.Sockets;
|
||||
using Server.Network;
|
||||
using Xunit;
|
||||
|
||||
namespace Server.Tests.Network;
|
||||
|
||||
[Collection("Sequential Server Tests")]
|
||||
public class NetworkStatsTests
|
||||
{
|
||||
private static byte[] Pattern(int length, int seed)
|
||||
{
|
||||
var data = new byte[length];
|
||||
new System.Random(seed).NextBytes(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MaintainSendBuffers_RecordsTheRefusalsItResets()
|
||||
{
|
||||
var previous = NetState._availableMemoryBytes;
|
||||
NetState ns = null;
|
||||
Socket client = null;
|
||||
|
||||
try
|
||||
{
|
||||
ns = PacketTestUtilities.CreateTestNetState(out client);
|
||||
ns._protocolState = NetState.ProtocolState.GameServer_AwaitingGameServerLogin;
|
||||
ns.Account = new MockAccount();
|
||||
var baseSize = ns._socket.SendBuffer.PhysicalSize;
|
||||
|
||||
NetState.MaintainSendBuffers(); // drain whatever earlier tests refused
|
||||
NetState._availableMemoryBytes = 1; // any working set is above 80% of one byte
|
||||
|
||||
for (var i = 0; i < 6; i++)
|
||||
{
|
||||
ns.Send(Pattern(baseSize / 4, i));
|
||||
}
|
||||
|
||||
// The sweep refreshes _availableMemoryBytes, so refusals must happen before it
|
||||
NetState.MaintainSendBuffers();
|
||||
var stats = NetState.GetNetworkStats();
|
||||
|
||||
Assert.True(stats.LastSweep.Ran);
|
||||
Assert.True(stats.LastSweep.CeilingRefusals >= 1);
|
||||
|
||||
// Counters are per sweep, not cumulative
|
||||
NetState.MaintainSendBuffers();
|
||||
Assert.Equal(0, NetState.GetNetworkStats().LastSweep.CeilingRefusals);
|
||||
}
|
||||
finally
|
||||
{
|
||||
NetState._availableMemoryBytes = previous;
|
||||
ns?.Dispose();
|
||||
client?.Close();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetNetworkStats_CountsAuthenticatedConnectionsSeparately()
|
||||
{
|
||||
var ns = PacketTestUtilities.CreateTestNetState(out var client);
|
||||
|
||||
try
|
||||
{
|
||||
var before = NetState.GetNetworkStats();
|
||||
ns._protocolState = NetState.ProtocolState.GameServer_AwaitingGameServerLogin;
|
||||
ns.Account = new MockAccount();
|
||||
var after = NetState.GetNetworkStats();
|
||||
|
||||
Assert.Equal(before.Connected, after.Connected);
|
||||
Assert.Equal(before.Authenticated + 1, after.Authenticated);
|
||||
Assert.True(after.Connected >= after.Authenticated);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ns.Dispose();
|
||||
client.Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -56,10 +56,7 @@ public partial class NetState
|
|||
internal static long _availableMemoryBytes;
|
||||
|
||||
private static Timer.DelayCallTimer _maintenanceTimer;
|
||||
private static long _lastTierCapacityBytes;
|
||||
private static int _lastTierInUse;
|
||||
private static int _lastTierRetainFloor;
|
||||
private static long _lastBaseCapacityBytes;
|
||||
private static SendBufferSweep _lastSweep;
|
||||
|
||||
private static readonly Queue<NetState> _disposed = [];
|
||||
private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds
|
||||
|
|
@ -238,14 +235,23 @@ public partial class NetState
|
|||
_capRefusals = 0;
|
||||
|
||||
var stats = _socketManager.Maintain();
|
||||
var changed = stats.TierCapacityBytes != _lastTierCapacityBytes ||
|
||||
stats.TierInUse != _lastTierInUse ||
|
||||
stats.TierRetainFloor != _lastTierRetainFloor ||
|
||||
stats.BaseCapacityBytes != _lastBaseCapacityBytes;
|
||||
_lastTierCapacityBytes = stats.TierCapacityBytes;
|
||||
_lastTierInUse = stats.TierInUse;
|
||||
_lastTierRetainFloor = stats.TierRetainFloor;
|
||||
_lastBaseCapacityBytes = stats.BaseCapacityBytes;
|
||||
var changed = stats.TierCapacityBytes != _lastSweep.TierCapacityBytes ||
|
||||
stats.TierInUse != _lastSweep.TierInUse ||
|
||||
stats.TierRetainFloor != _lastSweep.TierRetainFloor ||
|
||||
stats.BaseCapacityBytes != _lastSweep.BaseCapacityBytes;
|
||||
_lastSweep = new SendBufferSweep(
|
||||
true,
|
||||
Core.TickCount,
|
||||
stats.TierCapacityBytes,
|
||||
stats.TierInUse,
|
||||
stats.TierRetainFloor,
|
||||
stats.BuffersReleased,
|
||||
stats.BaseCapacityBytes,
|
||||
stats.BaseBuffersReleased,
|
||||
stats.GrowthRefusals,
|
||||
capRefusals,
|
||||
ceilingRefusals
|
||||
);
|
||||
|
||||
// Quiet unless something moved
|
||||
if (changed || stats.BuffersReleased > 0 || stats.BaseBuffersReleased > 0 || stats.GrowthRefusals > 0 ||
|
||||
|
|
@ -266,6 +272,48 @@ public partial class NetState
|
|||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Read-only diagnostics snapshot; never rotates the maintenance window. Cold path: allocates.
|
||||
/// </summary>
|
||||
public static NetworkStats GetNetworkStats()
|
||||
{
|
||||
var manager = _socketManager;
|
||||
var tiers = manager == null ? [] : new SendBufferTierUsage[manager.SendBufferTierCount];
|
||||
for (var i = 0; i < tiers.Length; i++)
|
||||
{
|
||||
var tier = manager.GetSendBufferTierStats(i);
|
||||
tiers[i] = new SendBufferTierUsage(tier.BufferSize, tier.Capacity, tier.InUse, tier.RetainFloor);
|
||||
}
|
||||
|
||||
var authenticated = 0;
|
||||
foreach (var ns in _instances)
|
||||
{
|
||||
if (ns.Account != null)
|
||||
{
|
||||
authenticated++;
|
||||
}
|
||||
}
|
||||
|
||||
return new NetworkStats(
|
||||
manager?.ConnectedCount ?? 0,
|
||||
manager?.MaxSockets ?? MaxConnections,
|
||||
authenticated,
|
||||
RecvBufferSize,
|
||||
SendBufferSize,
|
||||
MaxSendBufferSize,
|
||||
InitialRecvBufferSize,
|
||||
InitialSendBufferSize,
|
||||
_sendBufferGrowthBudget,
|
||||
_memoryCeilingPercent,
|
||||
_availableMemoryBytes,
|
||||
tiers,
|
||||
_lastSweep,
|
||||
_throttled.Count + _throttledPending.Count,
|
||||
_flushPending.Count,
|
||||
_pendingDisconnects.Count
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Slabs each base pool is divided into, after the transport applies its minimum slab size.
|
||||
/// </summary>
|
||||
|
|
|
|||
91
Projects/Server/Network/NetworkStats.cs
Normal file
91
Projects/Server/Network/NetworkStats.cs
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2026 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: NetworkStats.cs *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* You should have received a copy of the GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
*************************************************************************/
|
||||
|
||||
namespace Server.Network;
|
||||
|
||||
/// <summary>One send buffer growth tier's pool usage, in buffers of <paramref name="BufferSize"/> bytes.</summary>
|
||||
/// <param name="BufferSize">Physical size of every buffer in this tier.</param>
|
||||
/// <param name="Capacity">Buffers the tier's allocated slabs hold.</param>
|
||||
/// <param name="InUse">Buffers currently handed out.</param>
|
||||
/// <param name="RetainFloor">Peak usage the retention window is still holding capacity for.</param>
|
||||
public readonly record struct SendBufferTierUsage(int BufferSize, int Capacity, int InUse, int RetainFloor);
|
||||
|
||||
/// <summary>
|
||||
/// What the last once-a-minute send buffer maintenance sweep saw. The refusal counts cover the
|
||||
/// minute before that sweep only.
|
||||
/// </summary>
|
||||
/// <param name="Ran">False until the first sweep; every other field is then meaningless.</param>
|
||||
/// <param name="Tick"><see cref="Core.TickCount"/> when the sweep ran; compare by subtraction only.</param>
|
||||
/// <param name="TierCapacityBytes">Slab capacity allocated across the growth tiers.</param>
|
||||
/// <param name="TierInUse">Growth-tier buffers handed out, summed across tiers.</param>
|
||||
/// <param name="TierRetainFloor">Retention floors summed across tiers, in buffers.</param>
|
||||
/// <param name="TierBuffersReleased">Growth-tier buffers the sweep returned to the OS.</param>
|
||||
/// <param name="BaseCapacityBytes">Slab capacity allocated across the base and pre-auth pools.</param>
|
||||
/// <param name="BaseBuffersReleased">Base and pre-auth buffers the sweep returned to the OS.</param>
|
||||
/// <param name="BudgetRefusals">Growth refused because the tiers were at the growth budget.</param>
|
||||
/// <param name="CapRefusals">Growth refused because the buffer was already at the maximum size.</param>
|
||||
/// <param name="CeilingRefusals">Growth refused because the process was above the memory ceiling.</param>
|
||||
public readonly record struct SendBufferSweep(
|
||||
bool Ran,
|
||||
long Tick,
|
||||
long TierCapacityBytes,
|
||||
int TierInUse,
|
||||
int TierRetainFloor,
|
||||
int TierBuffersReleased,
|
||||
long BaseCapacityBytes,
|
||||
int BaseBuffersReleased,
|
||||
int BudgetRefusals,
|
||||
int CapRefusals,
|
||||
int CeilingRefusals
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Read-only snapshot of the network layer for diagnostics. Connection counts and tier usage are
|
||||
/// live; everything under <see cref="LastSweep"/> is up to a minute old. Cold path: allocates.
|
||||
/// </summary>
|
||||
/// <param name="Connected">Sockets the transport currently holds, authenticated or not.</param>
|
||||
/// <param name="MaxConnections">Socket slots the transport was built with.</param>
|
||||
/// <param name="Authenticated">Connections past login, and so holding full-size buffers.</param>
|
||||
/// <param name="RecvBufferSize">Base receive buffer size.</param>
|
||||
/// <param name="SendBufferSize">Base send buffer size.</param>
|
||||
/// <param name="MaxSendBufferSize">Largest size a send buffer can grow to.</param>
|
||||
/// <param name="InitialRecvBufferSize">Pre-auth receive buffer size; 0 when pre-auth buffers are off.</param>
|
||||
/// <param name="InitialSendBufferSize">Pre-auth send buffer size; 0 when pre-auth buffers are off.</param>
|
||||
/// <param name="SendBufferGrowthBudget">Bytes the growth tiers may hold in total.</param>
|
||||
/// <param name="MemoryCeilingPercent">Growth stops once the process exceeds this share of available memory.</param>
|
||||
/// <param name="AvailableMemoryBytes">Memory available to the process as of the last sweep.</param>
|
||||
/// <param name="Tiers">Live usage of each growth tier, smallest first.</param>
|
||||
/// <param name="LastSweep">What the last maintenance sweep saw.</param>
|
||||
/// <param name="Throttled">Connections whose parser is paused on a packet throttle.</param>
|
||||
/// <param name="FlushPending">Connections with unsent bytes queued for the next flush.</param>
|
||||
/// <param name="PendingDisconnects">Connections closing after their final flush.</param>
|
||||
public readonly record struct NetworkStats(
|
||||
int Connected,
|
||||
int MaxConnections,
|
||||
int Authenticated,
|
||||
int RecvBufferSize,
|
||||
int SendBufferSize,
|
||||
int MaxSendBufferSize,
|
||||
int InitialRecvBufferSize,
|
||||
int InitialSendBufferSize,
|
||||
long SendBufferGrowthBudget,
|
||||
int MemoryCeilingPercent,
|
||||
long AvailableMemoryBytes,
|
||||
SendBufferTierUsage[] Tiers,
|
||||
SendBufferSweep LastSweep,
|
||||
int Throttled,
|
||||
int FlushPending,
|
||||
int PendingDisconnects
|
||||
);
|
||||
|
|
@ -234,27 +234,16 @@ namespace Server.Gumps
|
|||
AddLabel(20, 130, LabelHue, "Event Loop:");
|
||||
AddLabel(40, 150, LabelHue, loopStatus);
|
||||
|
||||
using var sb = ValueStringBuilder.Create();
|
||||
var net = NetState.GetNetworkStats();
|
||||
|
||||
ThreadPool.GetAvailableThreads(out var curUser, out var curIOCP);
|
||||
ThreadPool.GetMaxThreads(out var maxUser, out var maxIOCP);
|
||||
AddLabel(20, 170, LabelHue, "Connections:");
|
||||
AddLabel(150, 170, LabelHue, $"{net.Connected} / {net.MaxConnections} ({net.Authenticated} authenticated)");
|
||||
|
||||
sb.Append("Worker Threads:<br>Capacity: ");
|
||||
sb.Append(maxUser);
|
||||
sb.Append("<br>Available: ");
|
||||
sb.Append(curUser);
|
||||
sb.Append("<br>Usage: ");
|
||||
sb.Append((maxUser - curUser) * 100 / maxUser);
|
||||
sb.Append("%<br><br>IOCP Threads:<br>Capacity: ");
|
||||
sb.Append(maxIOCP);
|
||||
sb.Append("<br>Available: ");
|
||||
sb.Append(curIOCP);
|
||||
sb.Append("<br>Usage: ");
|
||||
sb.Append((maxIOCP - curIOCP) * 100 / maxIOCP);
|
||||
sb.Append('%');
|
||||
AddLabel(20, 190, LabelHue, "Loop Queues:");
|
||||
AddLabel(150, 190, LabelHue, $"throttled {net.Throttled}, flush {net.FlushPending}, closing {net.PendingDisconnects}");
|
||||
|
||||
AddLabel(20, 200, LabelHue, "Pooling:");
|
||||
AddHtml(20, 220, 380, 150, sb.ToString(), true, true);
|
||||
AddLabel(20, 210, LabelHue, "Send Buffers:");
|
||||
AddHtml(20, 230, 380, 180, FormatSendBuffers(net), true, true);
|
||||
|
||||
AddPageButton(200, 20, GetButtonID(0, 0), "General", AdminGumpPage.Information_General);
|
||||
AddPageButton(200, 40, GetButtonID(0, 5), "Performance", AdminGumpPage.Information_Perf);
|
||||
|
|
@ -1364,6 +1353,48 @@ namespace Server.Gumps
|
|||
public static string FormatTimeSpan(TimeSpan ts) =>
|
||||
$"{ts.Days:D2}:{ts.Hours % 24:D2}:{ts.Minutes % 60:D2}:{ts.Seconds % 60:D2}";
|
||||
|
||||
private static string FormatSendBuffers(NetworkStats net)
|
||||
{
|
||||
using var sb = ValueStringBuilder.Create();
|
||||
|
||||
sb.Append($"Base: {FormatByteAmount(net.RecvBufferSize)} recv, {FormatByteAmount(net.SendBufferSize)} send");
|
||||
if (net.InitialRecvBufferSize > 0)
|
||||
{
|
||||
sb.Append($"; pre-auth {FormatByteAmount(net.InitialRecvBufferSize)} / {FormatByteAmount(net.InitialSendBufferSize)}");
|
||||
}
|
||||
|
||||
var sweep = net.LastSweep;
|
||||
if (sweep.Ran)
|
||||
{
|
||||
sb.Append($"<br>Base pools: {FormatByteAmount(sweep.BaseCapacityBytes)}");
|
||||
}
|
||||
|
||||
long tierBytes = 0;
|
||||
sb.Append("<br><br>Growth tiers (live):");
|
||||
for (var i = 0; i < net.Tiers.Length; i++)
|
||||
{
|
||||
var tier = net.Tiers[i];
|
||||
tierBytes += (long)tier.Capacity * tier.BufferSize;
|
||||
sb.Append($"<br> {FormatByteAmount(tier.BufferSize)}: {tier.InUse} / {tier.Capacity} in use, floor {tier.RetainFloor}");
|
||||
}
|
||||
|
||||
sb.Append($"<br>Budget: {FormatByteAmount(tierBytes)} of {FormatByteAmount(net.SendBufferGrowthBudget)}, max {FormatByteAmount(net.MaxSendBufferSize)} per socket");
|
||||
sb.Append($"<br>Ceiling: {net.MemoryCeilingPercent}% of {FormatByteAmount(net.AvailableMemoryBytes)} available");
|
||||
|
||||
if (sweep.Ran)
|
||||
{
|
||||
var ageSeconds = (Core.TickCount - sweep.Tick) / 1000;
|
||||
sb.Append($"<br><br>Last sweep {ageSeconds}s ago: released {sweep.TierBuffersReleased} tier, {sweep.BaseBuffersReleased} base");
|
||||
sb.Append($"<br>Refused: budget {sweep.BudgetRefusals}, at max {sweep.CapRefusals}, ceiling {sweep.CeilingRefusals}");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.Append("<br><br>First sweep pending");
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
public static string FormatByteAmount(long totalBytes)
|
||||
{
|
||||
return totalBytes switch
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue