From 80aff11a464ccd12b5c63ba8ffa6d8db6ee0669e Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:55:22 -0700 Subject: [PATCH] feat(network): consume IORingGroup 1.0.8 and expose send tuning IORingGroup 1.0.8 allows more than one send in flight per socket. RIO reports send completion on acknowledgement rather than on copy, so with a single send in flight a connection was capped at one send per round trip whenever it had data queued. On a 50ms-RTT shard that showed up as in-game latency of 100-145ms whenever any data was flowing, against a 50ms idle baseline; as little as 6 bytes of queued data was enough to hold the gate shut. Local testing could not surface it: the same measurement at loopback RTT is microseconds. Adds two restart-time settings: network.maxOutstandingSends (default 32) - sends in flight per connection. Honoured by RIO only; other backends complete sends on copy and report 1. Costs a request-queue and completion-queue slot per send, not another buffer, since every outstanding send addresses a different range of the same registered buffer. network.sendBufferSize (default 256KB) - per-connection send buffer, coerced to a power of two of at least the platform allocation granularity. This is the lever for "send buffer exhausted" disconnects and the per-connection memory ceiling. Co-Authored-By: Claude Opus 5 (1M context) --- .../Network/NetState/NetState.Network.cs | 57 +++++++++++++++++-- Projects/Server/Server.csproj | 2 +- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/Projects/Server/Network/NetState/NetState.Network.cs b/Projects/Server/Network/NetState/NetState.Network.cs index b38c88d23..5262c3c2e 100644 --- a/Projects/Server/Network/NetState/NetState.Network.cs +++ b/Projects/Server/Network/NetState/NetState.Network.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Net; using System.Net.NetworkInformation; using System.Network; +using System.Numerics; namespace Server.Network; @@ -28,9 +29,10 @@ namespace Server.Network; public partial class NetState { // Buffer sizes - private const int RecvBufferSize = 1024 * 64; // 64KB recv buffers - private const int SendBufferSize = 1024 * 256; // 256KB send buffers - private const int MaxConnections = 4096; // Max concurrent connections + private const int RecvBufferSize = 1024 * 64; // 64KB recv buffers + private const int DefaultSendBufferSize = 1024 * 256; // 256KB send buffers + private const int MinSendBufferSize = 1024 * 64; // Platform allocation granularity + private const int MaxConnections = 4096; // Max concurrent connections private static readonly Queue _disposed = []; private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds @@ -41,7 +43,9 @@ public partial class NetState // NetState storage indexed by RingSocket.Id private static readonly NetState[] _netStates = new NetState[MaxConnections]; - // Events buffer for ProcessCompletions + // Events buffer for ProcessCompletions. Bounded by one event per peeked completion + // (maxSockets), doubled for headroom. Undersizing drops DataReceived events whose bytes were + // already committed, leaving them unparsed until the next recv completes. private static readonly RingSocketEvent[] _events = new RingSocketEvent[MaxConnections * 2]; // Listener management @@ -88,20 +92,61 @@ public partial class NetState // Initialize IP rate limiter _ipRateLimiter = new IPRateLimiter(10, 10000, 1000, 2.0, 3_600_000, Core.ClosingTokenSource.Token); + // Sends in flight per connection; honoured by RIO only (see IIORingGroup). Costs a + // request-queue and completion-queue slot per send, not another buffer. Worst-case added + // latency is roughly completion RTT / this value. + var maxOutstandingSends = ServerConfiguration.GetOrUpdateSetting("network.maxOutstandingSends", 32); + // Initialize IORingGroup - var ring = IORingGroup.Create(queueSize: MaxConnections * 2, maxConnections: MaxConnections); + var ring = IORingGroup.Create( + queueSize: MaxConnections * 2, + maxConnections: MaxConnections, + maxOutstandingSends: maxOutstandingSends + ); + + // Per-connection send buffer: the lever for "send buffer exhausted" disconnects, and the + // per-connection memory ceiling. + var sendBufferSize = GetSendBufferSize(); // Create socket manager which handles buffer pools and socket lifecycle _socketManager = new RingSocketManager( ring, maxSockets: MaxConnections, recvBufferSize: RecvBufferSize, - sendBufferSize: SendBufferSize, + sendBufferSize: sendBufferSize, initialBufferSlabs: 8, maxBufferSlabs: 32 ); } + /// + /// Reads the configured send buffer size, coerced to a power of two of at least the platform + /// allocation granularity. IORingBuffer requires this and would otherwise throw at socket + /// creation rather than at startup. + /// + private static int GetSendBufferSize() + { + var configured = ServerConfiguration.GetOrUpdateSetting("network.sendBufferSize", DefaultSendBufferSize); + var size = Math.Max(MinSendBufferSize, configured); + + if (!BitOperations.IsPow2(size)) + { + size = (int)BitOperations.RoundUpToPowerOf2((uint)size); + } + + if (size != configured) + { + logger.Warning( + "network.sendBufferSize {Configured} is not a power of two of at least {Minimum}; using {Adjusted}", + configured, + MinSendBufferSize, + size + ); + } + + return size; + } + /// /// Starts the network server on configured listening addresses. /// diff --git a/Projects/Server/Server.csproj b/Projects/Server/Server.csproj index ff9b3a30b..8953efcba 100644 --- a/Projects/Server/Server.csproj +++ b/Projects/Server/Server.csproj @@ -34,7 +34,7 @@ - +