diff --git a/Projects/Server.Tests/Tests/Network/NetworkStatsTests.cs b/Projects/Server.Tests/Tests/Network/NetworkStatsTests.cs new file mode 100644 index 000000000..5a64a61fe --- /dev/null +++ b/Projects/Server.Tests/Tests/Network/NetworkStatsTests.cs @@ -0,0 +1,80 @@ +using System.Net.Sockets; +using Server.Network; +using Xunit; + +namespace Server.Tests.Network; + +[Collection("Sequential Server Tests")] +public class NetworkStatsTests +{ + private static byte[] Pattern(int length, int seed) + { + var data = new byte[length]; + new System.Random(seed).NextBytes(data); + return data; + } + + [Fact] + public void MaintainSendBuffers_RecordsTheRefusalsItResets() + { + var previous = NetState._availableMemoryBytes; + NetState ns = null; + Socket client = null; + + try + { + ns = PacketTestUtilities.CreateTestNetState(out client); + ns._protocolState = NetState.ProtocolState.GameServer_AwaitingGameServerLogin; + ns.Account = new MockAccount(); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + NetState.MaintainSendBuffers(); // drain whatever earlier tests refused + NetState._availableMemoryBytes = 1; // any working set is above 80% of one byte + + for (var i = 0; i < 6; i++) + { + ns.Send(Pattern(baseSize / 4, i)); + } + + // The sweep refreshes _availableMemoryBytes, so refusals must happen before it + NetState.MaintainSendBuffers(); + var stats = NetState.GetNetworkStats(); + + Assert.True(stats.LastSweep.Ran); + Assert.True(stats.LastSweep.CeilingRefusals >= 1); + + // Counters are per sweep, not cumulative + NetState.MaintainSendBuffers(); + Assert.Equal(0, NetState.GetNetworkStats().LastSweep.CeilingRefusals); + } + finally + { + NetState._availableMemoryBytes = previous; + ns?.Dispose(); + client?.Close(); + } + } + + [Fact] + public void GetNetworkStats_CountsAuthenticatedConnectionsSeparately() + { + var ns = PacketTestUtilities.CreateTestNetState(out var client); + + try + { + var before = NetState.GetNetworkStats(); + ns._protocolState = NetState.ProtocolState.GameServer_AwaitingGameServerLogin; + ns.Account = new MockAccount(); + var after = NetState.GetNetworkStats(); + + Assert.Equal(before.Connected, after.Connected); + Assert.Equal(before.Authenticated + 1, after.Authenticated); + Assert.True(after.Connected >= after.Authenticated); + } + finally + { + ns.Dispose(); + client.Close(); + } + } +} diff --git a/Projects/Server/Network/NetState/NetState.Network.cs b/Projects/Server/Network/NetState/NetState.Network.cs index 287fa78a4..f100f85c3 100644 --- a/Projects/Server/Network/NetState/NetState.Network.cs +++ b/Projects/Server/Network/NetState/NetState.Network.cs @@ -56,10 +56,7 @@ public partial class NetState internal static long _availableMemoryBytes; private static Timer.DelayCallTimer _maintenanceTimer; - private static long _lastTierCapacityBytes; - private static int _lastTierInUse; - private static int _lastTierRetainFloor; - private static long _lastBaseCapacityBytes; + private static SendBufferSweep _lastSweep; private static readonly Queue _disposed = []; private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds @@ -238,14 +235,23 @@ public partial class NetState _capRefusals = 0; var stats = _socketManager.Maintain(); - var changed = stats.TierCapacityBytes != _lastTierCapacityBytes || - stats.TierInUse != _lastTierInUse || - stats.TierRetainFloor != _lastTierRetainFloor || - stats.BaseCapacityBytes != _lastBaseCapacityBytes; - _lastTierCapacityBytes = stats.TierCapacityBytes; - _lastTierInUse = stats.TierInUse; - _lastTierRetainFloor = stats.TierRetainFloor; - _lastBaseCapacityBytes = stats.BaseCapacityBytes; + var changed = stats.TierCapacityBytes != _lastSweep.TierCapacityBytes || + stats.TierInUse != _lastSweep.TierInUse || + stats.TierRetainFloor != _lastSweep.TierRetainFloor || + stats.BaseCapacityBytes != _lastSweep.BaseCapacityBytes; + _lastSweep = new SendBufferSweep( + true, + Core.TickCount, + stats.TierCapacityBytes, + stats.TierInUse, + stats.TierRetainFloor, + stats.BuffersReleased, + stats.BaseCapacityBytes, + stats.BaseBuffersReleased, + stats.GrowthRefusals, + capRefusals, + ceilingRefusals + ); // Quiet unless something moved if (changed || stats.BuffersReleased > 0 || stats.BaseBuffersReleased > 0 || stats.GrowthRefusals > 0 || @@ -266,6 +272,48 @@ public partial class NetState } } + /// + /// Read-only diagnostics snapshot; never rotates the maintenance window. Cold path: allocates. + /// + public static NetworkStats GetNetworkStats() + { + var manager = _socketManager; + var tiers = manager == null ? [] : new SendBufferTierUsage[manager.SendBufferTierCount]; + for (var i = 0; i < tiers.Length; i++) + { + var tier = manager.GetSendBufferTierStats(i); + tiers[i] = new SendBufferTierUsage(tier.BufferSize, tier.Capacity, tier.InUse, tier.RetainFloor); + } + + var authenticated = 0; + foreach (var ns in _instances) + { + if (ns.Account != null) + { + authenticated++; + } + } + + return new NetworkStats( + manager?.ConnectedCount ?? 0, + manager?.MaxSockets ?? MaxConnections, + authenticated, + RecvBufferSize, + SendBufferSize, + MaxSendBufferSize, + InitialRecvBufferSize, + InitialSendBufferSize, + _sendBufferGrowthBudget, + _memoryCeilingPercent, + _availableMemoryBytes, + tiers, + _lastSweep, + _throttled.Count + _throttledPending.Count, + _flushPending.Count, + _pendingDisconnects.Count + ); + } + /// /// Slabs each base pool is divided into, after the transport applies its minimum slab size. /// diff --git a/Projects/Server/Network/NetworkStats.cs b/Projects/Server/Network/NetworkStats.cs new file mode 100644 index 000000000..370b8b7ff --- /dev/null +++ b/Projects/Server/Network/NetworkStats.cs @@ -0,0 +1,91 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: NetworkStats.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +namespace Server.Network; + +/// One send buffer growth tier's pool usage, in buffers of bytes. +/// Physical size of every buffer in this tier. +/// Buffers the tier's allocated slabs hold. +/// Buffers currently handed out. +/// Peak usage the retention window is still holding capacity for. +public readonly record struct SendBufferTierUsage(int BufferSize, int Capacity, int InUse, int RetainFloor); + +/// +/// What the last once-a-minute send buffer maintenance sweep saw. The refusal counts cover the +/// minute before that sweep only. +/// +/// False until the first sweep; every other field is then meaningless. +/// when the sweep ran; compare by subtraction only. +/// Slab capacity allocated across the growth tiers. +/// Growth-tier buffers handed out, summed across tiers. +/// Retention floors summed across tiers, in buffers. +/// Growth-tier buffers the sweep returned to the OS. +/// Slab capacity allocated across the base and pre-auth pools. +/// Base and pre-auth buffers the sweep returned to the OS. +/// Growth refused because the tiers were at the growth budget. +/// Growth refused because the buffer was already at the maximum size. +/// Growth refused because the process was above the memory ceiling. +public readonly record struct SendBufferSweep( + bool Ran, + long Tick, + long TierCapacityBytes, + int TierInUse, + int TierRetainFloor, + int TierBuffersReleased, + long BaseCapacityBytes, + int BaseBuffersReleased, + int BudgetRefusals, + int CapRefusals, + int CeilingRefusals +); + +/// +/// Read-only snapshot of the network layer for diagnostics. Connection counts and tier usage are +/// live; everything under is up to a minute old. Cold path: allocates. +/// +/// Sockets the transport currently holds, authenticated or not. +/// Socket slots the transport was built with. +/// Connections past login, and so holding full-size buffers. +/// Base receive buffer size. +/// Base send buffer size. +/// Largest size a send buffer can grow to. +/// Pre-auth receive buffer size; 0 when pre-auth buffers are off. +/// Pre-auth send buffer size; 0 when pre-auth buffers are off. +/// Bytes the growth tiers may hold in total. +/// Growth stops once the process exceeds this share of available memory. +/// Memory available to the process as of the last sweep. +/// Live usage of each growth tier, smallest first. +/// What the last maintenance sweep saw. +/// Connections whose parser is paused on a packet throttle. +/// Connections with unsent bytes queued for the next flush. +/// Connections closing after their final flush. +public readonly record struct NetworkStats( + int Connected, + int MaxConnections, + int Authenticated, + int RecvBufferSize, + int SendBufferSize, + int MaxSendBufferSize, + int InitialRecvBufferSize, + int InitialSendBufferSize, + long SendBufferGrowthBudget, + int MemoryCeilingPercent, + long AvailableMemoryBytes, + SendBufferTierUsage[] Tiers, + SendBufferSweep LastSweep, + int Throttled, + int FlushPending, + int PendingDisconnects +); diff --git a/Projects/UOContent/Gumps/AdminGump.cs b/Projects/UOContent/Gumps/AdminGump.cs index a4fbe0f18..8d75f15d6 100644 --- a/Projects/UOContent/Gumps/AdminGump.cs +++ b/Projects/UOContent/Gumps/AdminGump.cs @@ -234,27 +234,16 @@ namespace Server.Gumps AddLabel(20, 130, LabelHue, "Event Loop:"); AddLabel(40, 150, LabelHue, loopStatus); - using var sb = ValueStringBuilder.Create(); + var net = NetState.GetNetworkStats(); - ThreadPool.GetAvailableThreads(out var curUser, out var curIOCP); - ThreadPool.GetMaxThreads(out var maxUser, out var maxIOCP); + AddLabel(20, 170, LabelHue, "Connections:"); + AddLabel(150, 170, LabelHue, $"{net.Connected} / {net.MaxConnections} ({net.Authenticated} authenticated)"); - sb.Append("Worker Threads:
Capacity: "); - sb.Append(maxUser); - sb.Append("
Available: "); - sb.Append(curUser); - sb.Append("
Usage: "); - sb.Append((maxUser - curUser) * 100 / maxUser); - sb.Append("%

IOCP Threads:
Capacity: "); - sb.Append(maxIOCP); - sb.Append("
Available: "); - sb.Append(curIOCP); - sb.Append("
Usage: "); - sb.Append((maxIOCP - curIOCP) * 100 / maxIOCP); - sb.Append('%'); + AddLabel(20, 190, LabelHue, "Loop Queues:"); + AddLabel(150, 190, LabelHue, $"throttled {net.Throttled}, flush {net.FlushPending}, closing {net.PendingDisconnects}"); - AddLabel(20, 200, LabelHue, "Pooling:"); - AddHtml(20, 220, 380, 150, sb.ToString(), true, true); + AddLabel(20, 210, LabelHue, "Send Buffers:"); + AddHtml(20, 230, 380, 180, FormatSendBuffers(net), true, true); AddPageButton(200, 20, GetButtonID(0, 0), "General", AdminGumpPage.Information_General); AddPageButton(200, 40, GetButtonID(0, 5), "Performance", AdminGumpPage.Information_Perf); @@ -1364,6 +1353,48 @@ namespace Server.Gumps public static string FormatTimeSpan(TimeSpan ts) => $"{ts.Days:D2}:{ts.Hours % 24:D2}:{ts.Minutes % 60:D2}:{ts.Seconds % 60:D2}"; + private static string FormatSendBuffers(NetworkStats net) + { + using var sb = ValueStringBuilder.Create(); + + sb.Append($"Base: {FormatByteAmount(net.RecvBufferSize)} recv, {FormatByteAmount(net.SendBufferSize)} send"); + if (net.InitialRecvBufferSize > 0) + { + sb.Append($"; pre-auth {FormatByteAmount(net.InitialRecvBufferSize)} / {FormatByteAmount(net.InitialSendBufferSize)}"); + } + + var sweep = net.LastSweep; + if (sweep.Ran) + { + sb.Append($"
Base pools: {FormatByteAmount(sweep.BaseCapacityBytes)}"); + } + + long tierBytes = 0; + sb.Append("

Growth tiers (live):"); + for (var i = 0; i < net.Tiers.Length; i++) + { + var tier = net.Tiers[i]; + tierBytes += (long)tier.Capacity * tier.BufferSize; + sb.Append($"
{FormatByteAmount(tier.BufferSize)}: {tier.InUse} / {tier.Capacity} in use, floor {tier.RetainFloor}"); + } + + sb.Append($"
Budget: {FormatByteAmount(tierBytes)} of {FormatByteAmount(net.SendBufferGrowthBudget)}, max {FormatByteAmount(net.MaxSendBufferSize)} per socket"); + sb.Append($"
Ceiling: {net.MemoryCeilingPercent}% of {FormatByteAmount(net.AvailableMemoryBytes)} available"); + + if (sweep.Ran) + { + var ageSeconds = (Core.TickCount - sweep.Tick) / 1000; + sb.Append($"

Last sweep {ageSeconds}s ago: released {sweep.TierBuffersReleased} tier, {sweep.BaseBuffersReleased} base"); + sb.Append($"
Refused: budget {sweep.BudgetRefusals}, at max {sweep.CapRefusals}, ceiling {sweep.CeilingRefusals}"); + } + else + { + sb.Append("

First sweep pending"); + } + + return sb.ToString(); + } + public static string FormatByteAmount(long totalBytes) { return totalBytes switch