diff --git a/Projects/Server.Tests/Tests/Network/NetworkCompressionBoundsTests.cs b/Projects/Server.Tests/Tests/Network/NetworkCompressionBoundsTests.cs new file mode 100644 index 000000000..c1bc6a800 --- /dev/null +++ b/Projects/Server.Tests/Tests/Network/NetworkCompressionBoundsTests.cs @@ -0,0 +1,86 @@ +using System; +using Server.Network; +using Xunit; + +namespace Server.Tests.Network; + +/// +/// Bounds behaviour of the Huffman compressor when the destination is too small. +/// +/// This is reachable in production: NetState only checks that the send buffer has *some* writable +/// space before handing the remainder to Compress, so a nearly-full buffer can offer a span of one +/// to three bytes. The internal guard is computed as an unsigned output.Length - 4, which +/// underflows for those sizes and stops bounding the writes at all. +/// +public class NetworkCompressionBoundsTests +{ + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public void RefusesOutputTooSmallToBound(int outputSize) + { + var input = new byte[64]; + Array.Fill(input, (byte)'A'); + + // Sentinel-filled backing array; only the middle window is offered to the compressor, so + // any write past the span shows up as a modified sentinel rather than silent corruption. + var backing = new byte[256]; + Array.Fill(backing, (byte)0xCC); + + const int windowStart = 64; + var output = backing.AsSpan(windowStart, outputSize); + + var written = NetworkCompression.Compress(input, output); + + Assert.Equal(0, written); + + for (var i = 0; i < backing.Length; i++) + { + Assert.Equal(0xCC, backing[i]); + } + } + + [Fact] + public void StillCompressesWhenOutputIsLargeEnough() + { + var input = new byte[64]; + Array.Fill(input, (byte)'A'); + + var output = new byte[256]; + + var written = NetworkCompression.Compress(input, output); + + Assert.True(written > 0); + Assert.True(written <= output.Length); + } + + [Fact] + public void ReportsFailureRatherThanOverrunningATightOutput() + { + // Large input against a small-but-bounded output: the guard is well-defined here, so this + // must fail cleanly rather than write past the end. + var input = new byte[4096]; + Array.Fill(input, (byte)'A'); + + var backing = new byte[256]; + Array.Fill(backing, (byte)0xCC); + + const int windowStart = 64; + const int windowSize = 16; + var output = backing.AsSpan(windowStart, windowSize); + + NetworkCompression.Compress(input, output); + + for (var i = 0; i < windowStart; i++) + { + Assert.Equal(0xCC, backing[i]); + } + + for (var i = windowStart + windowSize; i < backing.Length; i++) + { + Assert.Equal(0xCC, backing[i]); + } + } +} diff --git a/Projects/Server/Network/NetState/NetState.Network.cs b/Projects/Server/Network/NetState/NetState.Network.cs index b38c88d23..5262c3c2e 100644 --- a/Projects/Server/Network/NetState/NetState.Network.cs +++ b/Projects/Server/Network/NetState/NetState.Network.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Net; using System.Net.NetworkInformation; using System.Network; +using System.Numerics; namespace Server.Network; @@ -28,9 +29,10 @@ namespace Server.Network; public partial class NetState { // Buffer sizes - private const int RecvBufferSize = 1024 * 64; // 64KB recv buffers - private const int SendBufferSize = 1024 * 256; // 256KB send buffers - private const int MaxConnections = 4096; // Max concurrent connections + private const int RecvBufferSize = 1024 * 64; // 64KB recv buffers + private const int DefaultSendBufferSize = 1024 * 256; // 256KB send buffers + private const int MinSendBufferSize = 1024 * 64; // Platform allocation granularity + private const int MaxConnections = 4096; // Max concurrent connections private static readonly Queue _disposed = []; private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds @@ -41,7 +43,9 @@ public partial class NetState // NetState storage indexed by RingSocket.Id private static readonly NetState[] _netStates = new NetState[MaxConnections]; - // Events buffer for ProcessCompletions + // Events buffer for ProcessCompletions. Bounded by one event per peeked completion + // (maxSockets), doubled for headroom. Undersizing drops DataReceived events whose bytes were + // already committed, leaving them unparsed until the next recv completes. private static readonly RingSocketEvent[] _events = new RingSocketEvent[MaxConnections * 2]; // Listener management @@ -88,20 +92,61 @@ public partial class NetState // Initialize IP rate limiter _ipRateLimiter = new IPRateLimiter(10, 10000, 1000, 2.0, 3_600_000, Core.ClosingTokenSource.Token); + // Sends in flight per connection; honoured by RIO only (see IIORingGroup). Costs a + // request-queue and completion-queue slot per send, not another buffer. Worst-case added + // latency is roughly completion RTT / this value. + var maxOutstandingSends = ServerConfiguration.GetOrUpdateSetting("network.maxOutstandingSends", 32); + // Initialize IORingGroup - var ring = IORingGroup.Create(queueSize: MaxConnections * 2, maxConnections: MaxConnections); + var ring = IORingGroup.Create( + queueSize: MaxConnections * 2, + maxConnections: MaxConnections, + maxOutstandingSends: maxOutstandingSends + ); + + // Per-connection send buffer: the lever for "send buffer exhausted" disconnects, and the + // per-connection memory ceiling. + var sendBufferSize = GetSendBufferSize(); // Create socket manager which handles buffer pools and socket lifecycle _socketManager = new RingSocketManager( ring, maxSockets: MaxConnections, recvBufferSize: RecvBufferSize, - sendBufferSize: SendBufferSize, + sendBufferSize: sendBufferSize, initialBufferSlabs: 8, maxBufferSlabs: 32 ); } + /// + /// Reads the configured send buffer size, coerced to a power of two of at least the platform + /// allocation granularity. IORingBuffer requires this and would otherwise throw at socket + /// creation rather than at startup. + /// + private static int GetSendBufferSize() + { + var configured = ServerConfiguration.GetOrUpdateSetting("network.sendBufferSize", DefaultSendBufferSize); + var size = Math.Max(MinSendBufferSize, configured); + + if (!BitOperations.IsPow2(size)) + { + size = (int)BitOperations.RoundUpToPowerOf2((uint)size); + } + + if (size != configured) + { + logger.Warning( + "network.sendBufferSize {Configured} is not a power of two of at least {Minimum}; using {Adjusted}", + configured, + MinSendBufferSize, + size + ); + } + + return size; + } + /// /// Starts the network server on configured listening addresses. /// diff --git a/Projects/Server/Network/NetState/NetState.cs b/Projects/Server/Network/NetState/NetState.cs index 6c6262ae4..05d8a8f16 100755 --- a/Projects/Server/Network/NetState/NetState.cs +++ b/Projects/Server/Network/NetState/NetState.cs @@ -444,17 +444,36 @@ public partial class NetState : IComparable, IValueLinkListNode buffer.Length) + { + SendBufferExhausted(span.Length, buffer.Length); + return; } else { @@ -484,6 +503,31 @@ public partial class NetState : IComparable, IValueLinkListNode + /// Handles a packet that cannot be placed in the send buffer. + /// + /// + /// High unacked means a slow client holding the buffer; needed approaching capacity means the + /// buffer is too small for this shard and network.sendBufferSize should be raised. + /// + private void SendBufferExhausted(int needed, int writable) + { + var sendBuffer = _socket?.SendBuffer; + var unacked = sendBuffer?.InFlightBytes ?? 0; + var capacity = sendBuffer?.PhysicalSize ?? 0; + + logger.Warning( + "{NetState}: send buffer exhausted - needed {Needed} bytes, {Writable} writable, {Unacked} awaiting acknowledgement, {Capacity} capacity. Raise network.sendBufferSize (power of two) if this recurs on healthy connections.", + this, + needed, + writable, + unacked, + capacity + ); + + Disconnect($"Send buffer exhausted (needed {needed}, writable {writable}, unacked {unacked}, capacity {capacity})"); + } + private void StartPacketLog() { try diff --git a/Projects/Server/Network/NetworkCompression.cs b/Projects/Server/Network/NetworkCompression.cs index 1a0154d40..12b4108da 100644 --- a/Projects/Server/Network/NetworkCompression.cs +++ b/Projects/Server/Network/NetworkCompression.cs @@ -73,7 +73,9 @@ public static class NetworkCompression public static int Compress(ReadOnlySpan input, Span output) { - if (input.Length > DefiniteOverflow) + // output.Length < 4 underflows safeOutputLength below (nuint), defeating the hot loop's + // bounds check. Reachable whenever the send buffer is nearly full. + if (input.Length > DefiniteOverflow || output.Length < 4) { return 0; } diff --git a/Projects/Server/Server.csproj b/Projects/Server/Server.csproj index ff9b3a30b..8953efcba 100644 --- a/Projects/Server/Server.csproj +++ b/Projects/Server/Server.csproj @@ -34,7 +34,7 @@ - +