feat(network): consume IORingGroup 1.0.8 and expose send tuning

IORingGroup 1.0.8 allows more than one send in flight per socket. RIO
reports send completion on acknowledgement rather than on copy, so with a
single send in flight a connection was capped at one send per round trip
whenever it had data queued. On a 50ms-RTT shard that showed up as in-game
latency of 100-145ms whenever any data was flowing, against a 50ms idle
baseline; as little as 6 bytes of queued data was enough to hold the gate
shut. Local testing could not surface it: the same measurement at loopback
RTT is microseconds.

Adds two restart-time settings:

  network.maxOutstandingSends (default 32) - sends in flight per
  connection. Honoured by RIO only; other backends complete sends on copy
  and report 1. Costs a request-queue and completion-queue slot per send,
  not another buffer, since every outstanding send addresses a different
  range of the same registered buffer.

  network.sendBufferSize (default 256KB) - per-connection send buffer,
  coerced to a power of two of at least the platform allocation
  granularity. This is the lever for "send buffer exhausted" disconnects
  and the per-connection memory ceiling.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kamron Batman 2026-07-27 22:55:22 -07:00
parent c909ed1f2f
commit 80aff11a46
No known key found for this signature in database
GPG key ID: 7D81DF26D9A5D94A
2 changed files with 52 additions and 7 deletions

View file

@ -19,6 +19,7 @@ using System.Linq;
using System.Net;
using System.Net.NetworkInformation;
using System.Network;
using System.Numerics;
namespace Server.Network;
@ -28,9 +29,10 @@ namespace Server.Network;
public partial class NetState
{
// Buffer sizes
private const int RecvBufferSize = 1024 * 64; // 64KB recv buffers
private const int SendBufferSize = 1024 * 256; // 256KB send buffers
private const int MaxConnections = 4096; // Max concurrent connections
private const int RecvBufferSize = 1024 * 64; // 64KB recv buffers
private const int DefaultSendBufferSize = 1024 * 256; // 256KB send buffers
private const int MinSendBufferSize = 1024 * 64; // Platform allocation granularity
private const int MaxConnections = 4096; // Max concurrent connections
private static readonly Queue<NetState> _disposed = [];
private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds
@ -41,7 +43,9 @@ public partial class NetState
// NetState storage indexed by RingSocket.Id
private static readonly NetState[] _netStates = new NetState[MaxConnections];
// Events buffer for ProcessCompletions
// Events buffer for ProcessCompletions. Bounded by one event per peeked completion
// (maxSockets), doubled for headroom. Undersizing drops DataReceived events whose bytes were
// already committed, leaving them unparsed until the next recv completes.
private static readonly RingSocketEvent[] _events = new RingSocketEvent[MaxConnections * 2];
// Listener management
@ -88,20 +92,61 @@ public partial class NetState
// Initialize IP rate limiter
_ipRateLimiter = new IPRateLimiter(10, 10000, 1000, 2.0, 3_600_000, Core.ClosingTokenSource.Token);
// Sends in flight per connection; honoured by RIO only (see IIORingGroup). Costs a
// request-queue and completion-queue slot per send, not another buffer. Worst-case added
// latency is roughly completion RTT / this value.
var maxOutstandingSends = ServerConfiguration.GetOrUpdateSetting("network.maxOutstandingSends", 32);
// Initialize IORingGroup
var ring = IORingGroup.Create(queueSize: MaxConnections * 2, maxConnections: MaxConnections);
var ring = IORingGroup.Create(
queueSize: MaxConnections * 2,
maxConnections: MaxConnections,
maxOutstandingSends: maxOutstandingSends
);
// Per-connection send buffer: the lever for "send buffer exhausted" disconnects, and the
// per-connection memory ceiling.
var sendBufferSize = GetSendBufferSize();
// Create socket manager which handles buffer pools and socket lifecycle
_socketManager = new RingSocketManager(
ring,
maxSockets: MaxConnections,
recvBufferSize: RecvBufferSize,
sendBufferSize: SendBufferSize,
sendBufferSize: sendBufferSize,
initialBufferSlabs: 8,
maxBufferSlabs: 32
);
}
/// <summary>
/// Reads the configured send buffer size, coerced to a power of two of at least the platform
/// allocation granularity. IORingBuffer requires this and would otherwise throw at socket
/// creation rather than at startup.
/// </summary>
private static int GetSendBufferSize()
{
var configured = ServerConfiguration.GetOrUpdateSetting("network.sendBufferSize", DefaultSendBufferSize);
var size = Math.Max(MinSendBufferSize, configured);
if (!BitOperations.IsPow2(size))
{
size = (int)BitOperations.RoundUpToPowerOf2((uint)size);
}
if (size != configured)
{
logger.Warning(
"network.sendBufferSize {Configured} is not a power of two of at least {Minimum}; using {Adjusted}",
configured,
MinSendBufferSize,
size
);
}
return size;
}
/// <summary>
/// Starts the network server on configured listening addresses.
/// </summary>

View file

@ -34,7 +34,7 @@
</Target>
<ItemGroup>
<ProjectReference Include="..\Logger\Logger.csproj" />
<PackageReference Include="IORingGroup" Version="1.0.7" />
<PackageReference Include="IORingGroup" Version="1.0.8" />
<PackageReference Include="CommunityToolkit.HighPerformance" Version="8.4.2" />
<PackageReference Include="LibDeflate.Bindings" Version="1.0.3" />
<PackageReference Include="System.IO.Hashing" Version="10.0.10" />