From 31cd19b05b2ce2fe6950222457241eab438f95d5 Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:35:06 -0700 Subject: [PATCH] feat(network): grow the send buffer on demand instead of disconnecting (#2639) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A connection's send buffer is a fixed 256 KB. A burst of world traffic (a crowded area, a mass spawn, a war) that outruns the client's acknowledgements fills it, `NetState.Send()` reports "send buffer exhausted", and the player is disconnected. Raising the size for everyone multiplies the per-connection footprint (4096 × 256 KB is already 1 GB at full occupancy, page-locked on Windows). ## What changes - **Growth.** When a packet does not fit (the write span is too small, the packet is larger than the span, or compression returns 0), `Send()` asks the transport to grow the buffer to the next power-of-two tier and retries, up to `network.sendBufferMaxSize` (2 MB). Compression retries once per tier since its output size is not known in advance, including when the buffer is completely full. Only when growth is refused does the existing exhaustion disconnect run. The success path is unchanged. - **Memory ceiling.** Growth is refused (with a once-a-minute warning) when the process working set exceeds `network.memoryCeilingPercent` (80) of the memory available to the process (container-aware; `0` turns the check off). The figure is sampled at startup and refreshed each maintenance tick. - **Shrink.** A grown socket returns to the base buffer once it is drained and 30 s have passed since its last growth, attempted from the `DataSent` handler and from the 5 s alive sweep. - **Retention.** Every minute a timer calls the transport's `Maintain()`, which trims idle tier slabs down to the peak concurrent usage of the last 15 minutes, so recurring bursts reuse buffers without allocation while rare ones give the memory back. The line logs at Debug, and only when capacity, usage, or the floor changed or a growth was refused (budget, at max, or ceiling), so an idle shard logs nothing. - **Budget.** `network.sendBufferGrowthBudget` (256 MB) caps the tier pools' capacity; a positive value below one tier slab is raised with a warning, a negative one is clamped to 0 (growth off). Worst case is base × connections plus the budget. - Settings are coerced with accurate warnings (power of two, minimum, 256 MB transport ceiling). `[dumpnetstates` gains the send buffer size. `dev-docs/server-requirements.md` describes the new memory story. ## Tests `NetStateSendBufferTests` (real loopback sockets): growth instead of disconnect with a byte-exact stream, compressed growth against the compressor's own output, the grow-then-copy path, growth with a send genuinely in flight, refusal past the maximum, refusal under the ceiling, refusal on a closing socket, shrink after the hold (direct and through the alive sweep), and the setting coercions. Server.Tests 891 passed, UOContent.Tests 1052 passed against the published 1.0.12. Reviewed per task, whole-branch, and adversarially by a second model (twice, the second time jointly with the transport branch); all findings addressed. --- .../Tests/Network/NetStateDisconnectTests.cs | 8 +- .../Tests/Network/NetStateSendBufferTests.cs | 451 ++++++++++++++++++ .../Server/Network/NetState/DumpNetStates.cs | 4 +- .../Network/NetState/NetState.Network.cs | 175 ++++++- Projects/Server/Network/NetState/NetState.cs | 153 +++++- Projects/Server/Network/NetworkCompression.cs | 4 +- Projects/Server/Server.csproj | 2 +- dev-docs/server-requirements.md | 12 +- 8 files changed, 761 insertions(+), 48 deletions(-) create mode 100644 Projects/Server.Tests/Tests/Network/NetStateSendBufferTests.cs diff --git a/Projects/Server.Tests/Tests/Network/NetStateDisconnectTests.cs b/Projects/Server.Tests/Tests/Network/NetStateDisconnectTests.cs index adc513e2c..d1f273db3 100644 --- a/Projects/Server.Tests/Tests/Network/NetStateDisconnectTests.cs +++ b/Projects/Server.Tests/Tests/Network/NetStateDisconnectTests.cs @@ -225,12 +225,10 @@ public class NetStateDisconnectTests try { - var capacity = ns._socket.SendBuffer.PhysicalSize; + ns.Send(new byte[NetState.MaxSendBufferSize + 1]); // cannot fit; queues the disconnect + ns.Send(new byte[NetState.MaxSendBufferSize + 2]); // same tick; must not re-report - ns.Send(new byte[capacity + 1]); // cannot fit; queues the disconnect - ns.Send(new byte[capacity + 2]); // same tick; must not re-report - - Assert.Contains($"needed {capacity + 1}", ns._disconnectReason); + Assert.Contains($"needed {NetState.MaxSendBufferSize + 1}", ns._disconnectReason); } finally { diff --git a/Projects/Server.Tests/Tests/Network/NetStateSendBufferTests.cs b/Projects/Server.Tests/Tests/Network/NetStateSendBufferTests.cs new file mode 100644 index 000000000..2199052d2 --- /dev/null +++ b/Projects/Server.Tests/Tests/Network/NetStateSendBufferTests.cs @@ -0,0 +1,451 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net.Sockets; +using System.Network; +using System.Threading; +using Server.Network; +using Xunit; + +namespace Server.Tests.Network; + +[Collection("Sequential Server Tests")] +public class NetStateSendBufferTests +{ + private static NetState CreateAuthenticatedNetState(out Socket client) + { + var ns = PacketTestUtilities.CreateTestNetState(out client); + ns.Account = new MockAccount(); + return ns; + } + + private static byte[] Pattern(int length, int seed) + { + var data = new byte[length]; + new System.Random(seed).NextBytes(data); + return data; + } + + // A random prefix lands under the target; zeros (2 bits each) walk it up a byte at a time + private static byte[] CompressesToExactly(int compressedLength, int seed) + { + var randomLength = compressedLength * 5 / 8; + var data = new byte[compressedLength * 8]; + Pattern(randomLength, seed).CopyTo(data, 0); + + var scratch = new byte[compressedLength * 2 + 16]; + for (var length = randomLength; length < data.Length; length++) + { + var compressed = NetworkCompression.Compress(data.AsSpan(0, length), scratch); + Assert.InRange(compressed, 1, compressedLength); + + if (compressed == compressedLength) + { + return data[..length]; + } + } + + throw new InvalidOperationException($"No chunk compresses to exactly {compressedLength} bytes"); + } + + private static byte[] ReadAll(Socket client, int length) + { + var received = new byte[length]; + var total = 0; + var deadline = Stopwatch.StartNew(); + while (total < length && deadline.ElapsedMilliseconds < 10000) + { + NetState.Slice(); + + // Poll takes microseconds, not milliseconds + if (client.Poll(50_000, SelectMode.SelectRead)) + { + var read = client.Receive(received, total, length - total, SocketFlags.None); + Assert.NotEqual(0, read); + total += read; + } + } + + Assert.Equal(length, total); + return received; + } + + // Peer receipt does not free the buffer; the completion must land first + private static void WaitForDrain(NetState ns) + { + var deadline = Stopwatch.StartNew(); + while (deadline.ElapsedMilliseconds < 10000) + { + NetState.Slice(); + + if (ns._socket.SendBuffer.ReadableBytes == 0 && ns._socket.SendBuffer.InFlightBytes == 0) + { + break; + } + + Thread.Sleep(5); + } + + Assert.Equal(0, ns._socket.SendBuffer.ReadableBytes); + Assert.Equal(0, ns._socket.SendBuffer.InFlightBytes); + } + + [Fact] + public void Send_GrowsInsteadOfDisconnecting_AndStreamStaysIntact() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + // No Slice(): nothing drains, so the base buffer overflows + var chunk = baseSize / 4; + var expected = new byte[chunk * 6]; + for (var i = 0; i < 6; i++) + { + var data = Pattern(chunk, i); + data.CopyTo(expected, i * chunk); + ns.Send(data); + } + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.True(ns._socket.SendBuffer.PhysicalSize > baseSize); + Assert.True(ns._sendBufferGrown); + + Assert.Equal(expected, ReadAll(client, expected.Length)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_CompressedGrowth_RetriesAndStreamStaysIntact() + { + var ns = CreateAuthenticatedNetState(out var client); + ns.CompressionEnabled = true; + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + // No Slice(): compressed output accumulates until the base overflows + var chunkLength = baseSize / 4; + var expected = new byte[(chunkLength * 2 + 4) * 6]; + var scratch = new byte[chunkLength * 2 + 4]; + var expectedLength = 0; + for (var i = 0; i < 6; i++) + { + var data = Pattern(chunkLength, i + 100); + var compressedLength = NetworkCompression.Compress(data, scratch); + Assert.True(compressedLength > 0); + scratch.AsSpan(0, compressedLength).CopyTo(expected.AsSpan(expectedLength)); + expectedLength += compressedLength; + ns.Send(data); + } + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.True(ns._socket.SendBuffer.PhysicalSize > baseSize); + Assert.True(ns._sendBufferGrown); + + Array.Resize(ref expected, expectedLength); + Assert.Equal(expected, ReadAll(client, expected.Length)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_CompressedGrowth_FullBuffer_GrowsOneTierAndFits() + { + var ns = CreateAuthenticatedNetState(out var client); + ns.CompressionEnabled = true; + var baseSize = ns._socket.SendBuffer.PhysicalSize; + var expected = new List(baseSize * 2); + + void SendAndRecord(byte[] data) + { + var scratch = new byte[data.Length * 2 + 4]; + var compressedLength = NetworkCompression.Compress(data, scratch); + Assert.True(compressedLength > 0); + expected.AddRange(scratch.AsSpan(0, compressedLength)); + ns.Send(data); + } + + try + { + // Chunks sized so the fill cannot overflow (random bytes never compress; ratio at most 11/8) + var seed = 600; + while (ns._socket.SendBuffer.WritableBytes > 4096) + { + var writable = ns._socket.SendBuffer.WritableBytes; + SendAndRecord(Pattern(Math.Min(baseSize / 8, (writable - 2048) * 8 / 11), seed++)); + } + + // Full; no span to compress into + SendAndRecord(CompressesToExactly(ns._socket.SendBuffer.WritableBytes, seed)); + Assert.Equal(0, ns._socket.SendBuffer.WritableBytes); + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + + // Raw exceeds the remainder; compressed fits in one tier + SendAndRecord(new byte[Math.Min(baseSize * 3 / 4, NetworkCompression.DefiniteOverflow)]); + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.Equal(baseSize * 2, ns._socket.SendBuffer.PhysicalSize); + Assert.True(ns._sendBufferGrown); + + Assert.Equal(expected.ToArray(), ReadAll(client, expected.Count)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_WithPartialRemainder_GrowsAndCopiesFullPayload() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + var first = Pattern(baseSize / 2, 200); + var second = Pattern(baseSize / 2 + 1, 201); + var expected = new byte[first.Length + second.Length]; + first.CopyTo(expected, 0); + second.CopyTo(expected, first.Length); + + // Remainder too small for the second send + ns.Send(first); + ns.Send(second); + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.True(ns._socket.SendBuffer.PhysicalSize > baseSize); + Assert.True(ns._sendBufferGrown); + + Assert.Equal(expected, ReadAll(client, expected.Length)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_WithSendInFlight_GrowsAndStreamStaysIntact() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + // A small peer buffer keeps the first send in flight through the growth + client.ReceiveBufferSize = 4096; + + var first = Pattern(baseSize / 2, 300); + var second = Pattern(baseSize / 2 + 1, 301); + + ns.Send(first); + NetState.Slice(); // posts the first send + Assert.True(ns._socket.SendBuffer.InFlightBytes > 0); + + ns.Send(second); + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.True(ns._socket.SendBuffer.PhysicalSize > baseSize); + Assert.True(ns._sendBufferGrown); + + var expected = new byte[first.Length + second.Length]; + first.CopyTo(expected, 0); + second.CopyTo(expected, first.Length); + + Assert.Equal(expected, ReadAll(client, expected.Length)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Shrink_AfterDrainAndHold_ReturnsToBase() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + var chunk = baseSize / 4; + for (var i = 0; i < 6; i++) + { + ns.Send(Pattern(chunk, i)); + } + + Assert.True(ns._sendBufferGrown); + ReadAll(client, chunk * 6); + WaitForDrain(ns); + + var grewAt = ns._sendBufferGrewAt; + Assert.False(ns.TryShrinkSendBuffer(grewAt + NetState.SendBufferHoldMs - 1)); + Assert.True(ns.TryShrinkSendBuffer(grewAt + NetState.SendBufferHoldMs)); + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + Assert.False(ns._sendBufferGrown); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Shrink_ThroughAliveSweep_ReturnsToBase() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + var previousTicks = Core._tickCount; + + try + { + var chunk = baseSize / 4; + for (var i = 0; i < 6; i++) + { + ns.Send(Pattern(chunk, i + 400)); + } + + Assert.True(ns._sendBufferGrown); + ReadAll(client, chunk * 6); + WaitForDrain(ns); + + // The sweep shrinks; nothing here calls TryShrinkSendBuffer + Core._tickCount = ns._sendBufferGrewAt + NetState.SendBufferHoldMs; + NetState.CheckAllAlive(); + + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + Assert.False(ns._sendBufferGrown); + } + finally + { + Core._tickCount = previousTicks; + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_PastTheMaximum_FallsBackToExhaustion() + { + var ns = CreateAuthenticatedNetState(out var client); + + try + { + var chunk = 64 * 1024; + var writes = NetState.MaxSendBufferSize / chunk + 1; + for (var i = 0; i < writes; i++) + { + ns.Send(Pattern(chunk, i)); + } + + Assert.Contains("Send buffer exhausted", ns._disconnectReason); + Assert.Equal(NetState.MaxSendBufferSize, ns._socket.SendBuffer.PhysicalSize); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_AboveMemoryCeiling_DoesNotGrow() + { + var previous = NetState._availableMemoryBytes; + NetState ns = null; + Socket client = null; + + try + { + ns = CreateAuthenticatedNetState(out client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + NetState._availableMemoryBytes = 1; // any working set is above 80% of one byte + + for (var i = 0; i < 6; i++) + { + ns.Send(Pattern(baseSize / 4, i)); + } + + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + Assert.Contains("Send buffer exhausted", ns._disconnectReason); + } + finally + { + NetState._availableMemoryBytes = previous; + ns?.Dispose(); + client?.Close(); + } + } + + [Fact] + public void Send_OnClosingSocket_DoesNotGrow() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + ns.Disconnect("test"); + NetState.Slice(); // handoff + + // CannotSendPackets short-circuits before any growth + ns.Send(Pattern(baseSize * 2, 500)); + + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + Assert.False(ns._sendBufferGrown); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Theory] + [InlineData(3 * 1024 * 1024, 4 * 1024 * 1024)] // not a power of two; rounded up + [InlineData(512 * 1024 * 1024, 256 * 1024 * 1024)] // above the transport ceiling; capped + [InlineData(1024, 64 * 1024)] // below the minimum; raised + [InlineData(2 * 1024 * 1024, 2 * 1024 * 1024)] // already valid; untouched + public void CoercePowerOfTwoSetting_CoercesToATierTheTransportAccepts(int configured, int expected) => + Assert.Equal(expected, NetState.CoercePowerOfTwoSetting("network.sendBufferMaxSize", configured, 64 * 1024)); + + [Fact] + public void CoerceSendBufferGrowthBudget_ClampsNegativeAndRaisesBelowOneSlab() + { + const int sendBufferSize = 256 * 1024; + const int maxSendBufferSize = 2 * 1024 * 1024; + var minimum = RingSocketManager.MinimumSendBufferGrowthBudget(sendBufferSize); + + Assert.Equal(0L, NetState.CoerceSendBufferGrowthBudget(-1, sendBufferSize, maxSendBufferSize)); + + // zero means never grow + Assert.Equal(0L, NetState.CoerceSendBufferGrowthBudget(0, sendBufferSize, maxSendBufferSize)); + + Assert.Equal(minimum, NetState.CoerceSendBufferGrowthBudget(1, sendBufferSize, maxSendBufferSize)); + Assert.Equal(minimum * 4, NetState.CoerceSendBufferGrowthBudget(minimum * 4, sendBufferSize, maxSendBufferSize)); + + // growth off; budget untouched + Assert.Equal(1L, NetState.CoerceSendBufferGrowthBudget(1, sendBufferSize, sendBufferSize)); + } +} diff --git a/Projects/Server/Network/NetState/DumpNetStates.cs b/Projects/Server/Network/NetState/DumpNetStates.cs index 018ebb7e1..1dae56203 100644 --- a/Projects/Server/Network/NetState/DumpNetStates.cs +++ b/Projects/Server/Network/NetState/DumpNetStates.cs @@ -28,11 +28,11 @@ public static class DumpNetStates { using var file = new StreamWriter($"netstatedump-{Core.Now:yyyy-M-d-HH-mm-ss}_{Core.TickCount}.csv"); - file.WriteLine("NetState, ConnectedOn, NextActivityCheck, SocketConnected, ProtocolState, ParserState"); + file.WriteLine("NetState, ConnectedOn, NextActivityCheck, SocketConnected, ProtocolState, ParserState, SendBufferSize"); foreach (var ns in NetState.Instances) { - file.WriteLine($"{ns}, {ns.ConnectedOn}, {ns.NextActivityCheck}, {ns.IsConnected}, {ns._protocolState}, {ns._parserState}"); + file.WriteLine($"{ns}, {ns.ConnectedOn}, {ns.NextActivityCheck}, {ns.IsConnected}, {ns._protocolState}, {ns._parserState}, {ns._socket?.SendBuffer.PhysicalSize ?? 0}"); } } } diff --git a/Projects/Server/Network/NetState/NetState.Network.cs b/Projects/Server/Network/NetState/NetState.Network.cs index 5bedb504c..e393446b1 100644 --- a/Projects/Server/Network/NetState/NetState.Network.cs +++ b/Projects/Server/Network/NetState/NetState.Network.cs @@ -32,8 +32,26 @@ public partial class NetState private const int RecvBufferSize = 1024 * 64; // 64KB recv buffers private const int DefaultSendBufferSize = 1024 * 256; // 256KB send buffers private const int MinSendBufferSize = 1024 * 64; // Platform allocation granularity + private const int DefaultMaxSendBufferSize = 1024 * 1024 * 2; // 2 MB + private const long DefaultSendBufferGrowthBudget = 1024L * 1024 * 256; // 256 MB + private const int DefaultMemoryCeilingPercent = 80; + + // Transport ceiling; larger values overflow its tier enumeration + private const int TransportMaxSendBufferSize = 1024 * 1024 * 256; // 256 MB private const int MaxConnections = 4096; // Max concurrent connections + internal static int MaxSendBufferSize { get; private set; } + private static long _sendBufferGrowthBudget; + private static int _memoryCeilingPercent; + + // Refreshed by the maintenance sweep; internal for tests + internal static long _availableMemoryBytes; + + private static Timer.DelayCallTimer _maintenanceTimer; + private static long _lastTierCapacityBytes; + private static int _lastTierInUse; + private static int _lastTierRetainFloor; + private static readonly Queue _disposed = []; private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds @@ -121,16 +139,26 @@ public partial class NetState // latency is roughly completion RTT / this value. var maxOutstandingSends = ServerConfiguration.GetOrUpdateSetting("network.maxOutstandingSends", 32); - // Initialize IORingGroup - var ring = IORingGroup.Create( - queueSize: MaxConnections * 2, - maxConnections: MaxConnections, - maxOutstandingSends: maxOutstandingSends - ); - // Per-connection send buffer: the lever for "send buffer exhausted" disconnects, and the // per-connection memory ceiling. var sendBufferSize = GetSendBufferSize(); + MaxSendBufferSize = GetPowerOfTwoSetting("network.sendBufferMaxSize", DefaultMaxSendBufferSize, sendBufferSize); + _sendBufferGrowthBudget = CoerceSendBufferGrowthBudget( + ServerConfiguration.GetOrUpdateSetting("network.sendBufferGrowthBudget", DefaultSendBufferGrowthBudget), + sendBufferSize, + MaxSendBufferSize + ); + // 0 disables the ceiling + _memoryCeilingPercent = Math.Clamp(ServerConfiguration.GetOrUpdateSetting("network.memoryCeilingPercent", DefaultMemoryCeilingPercent), 0, 100); + _availableMemoryBytes = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + + const int maxBufferSlabs = 32; + var ring = IORingGroup.Create( + queueSize: MaxConnections * 2, + maxConnections: MaxConnections, + maxOutstandingSends: maxOutstandingSends, + maxRegisteredBuffers: RingSocketManager.RequiredRegisteredBuffers(MaxConnections, sendBufferSize, MaxSendBufferSize, _sendBufferGrowthBudget, maxBufferSlabs) + ); // Create socket manager which handles buffer pools and socket lifecycle _socketManager = new RingSocketManager( @@ -139,8 +167,51 @@ public partial class NetState recvBufferSize: RecvBufferSize, sendBufferSize: sendBufferSize, initialBufferSlabs: 8, - maxBufferSlabs: 32 + maxBufferSlabs: maxBufferSlabs, + maxSendBufferSize: MaxSendBufferSize, + sendBufferGrowthBudget: _sendBufferGrowthBudget ); + + _maintenanceTimer = Timer.DelayCall(TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), MaintainSendBuffers); + } + + internal static void MaintainSendBuffers() + { + if (_socketManager == null) + { + return; + } + + // Container limits can change + _availableMemoryBytes = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + + var ceilingRefusals = _ceilingRefusals; + var capRefusals = _capRefusals; + _ceilingRefusals = 0; + _capRefusals = 0; + + var stats = _socketManager.Maintain(); + var changed = stats.TierCapacityBytes != _lastTierCapacityBytes || + stats.TierInUse != _lastTierInUse || + stats.TierRetainFloor != _lastTierRetainFloor; + _lastTierCapacityBytes = stats.TierCapacityBytes; + _lastTierInUse = stats.TierInUse; + _lastTierRetainFloor = stats.TierRetainFloor; + + // Quiet unless something moved + if (changed || stats.BuffersReleased > 0 || stats.GrowthRefusals > 0 || capRefusals > 0 || ceilingRefusals > 0) + { + logger.Debug( + "Send buffer tiers: {Capacity} bytes of tier capacity, {InUse} buffers in use, floor {Floor} buffers, released {Released}, refused: budget {BudgetRefusals}, at max {CapRefusals}, ceiling {CeilingRefusals}", + stats.TierCapacityBytes, + stats.TierInUse, + stats.TierRetainFloor, + stats.BuffersReleased, + stats.GrowthRefusals, + capRefusals, + ceilingRefusals + ); + } } /// @@ -148,29 +219,89 @@ public partial class NetState /// allocation granularity. IORingBuffer requires this and would otherwise throw at socket /// creation rather than at startup. /// - private static int GetSendBufferSize() + private static int GetSendBufferSize() => + GetPowerOfTwoSetting("network.sendBufferSize", DefaultSendBufferSize, MinSendBufferSize); + + private static int GetPowerOfTwoSetting(string key, int defaultValue, int minimum) => + CoercePowerOfTwoSetting(key, ServerConfiguration.GetOrUpdateSetting(key, defaultValue), minimum); + + /// + /// Clamps to a power of two between and the transport ceiling. + /// + internal static int CoercePowerOfTwoSetting(string key, int configured, int minimum) { - var configured = ServerConfiguration.GetOrUpdateSetting("network.sendBufferSize", DefaultSendBufferSize); - var size = Math.Max(MinSendBufferSize, configured); + var size = configured; + + if (size > TransportMaxSendBufferSize) + { + logger.Warning( + "{Key} {Configured} is above the transport maximum {Maximum} (capped); using {Adjusted}", + key, + configured, + TransportMaxSendBufferSize, + TransportMaxSendBufferSize + ); + + size = TransportMaxSendBufferSize; + } + + if (size < minimum) + { + logger.Warning( + "{Key} {Configured} is below the minimum {Minimum} (raised); using {Adjusted}", + key, + configured, + minimum, + minimum + ); + + size = minimum; + } if (!BitOperations.IsPow2(size)) { - size = (int)BitOperations.RoundUpToPowerOf2((uint)size); - } + // Rounding up cannot cross the ceiling, itself a power of two + var rounded = (int)BitOperations.RoundUpToPowerOf2((uint)size); - if (size != configured) - { - logger.Warning( - "network.sendBufferSize {Configured} is not a power of two of at least {Minimum}; using {Adjusted}", - configured, - MinSendBufferSize, - size - ); + logger.Warning("{Key} {Configured} is not a power of two; using {Adjusted}", key, configured, rounded); + size = rounded; } return size; } + /// + /// Negative disables growth; below one tier slab is raised to the minimum. + /// + internal static long CoerceSendBufferGrowthBudget(long configured, int sendBufferSize, int maxSendBufferSize) + { + if (configured < 0) + { + logger.Warning("network.sendBufferGrowthBudget {Configured} is negative; using 0 (growth disabled)", configured); + return 0; + } + + if (configured == 0 || maxSendBufferSize <= sendBufferSize) + { + return configured; + } + + // Tier buffers are allocated a slab at a time. + var minimumBudget = RingSocketManager.MinimumSendBufferGrowthBudget(sendBufferSize); + if (configured < minimumBudget) + { + logger.Warning( + "network.sendBufferGrowthBudget {Configured} is below one tier slab; using {Minimum}", + configured, + minimumBudget + ); + + return minimumBudget; + } + + return configured; + } + /// /// Starts the network server on configured listening addresses. /// @@ -494,6 +625,7 @@ public partial class NetState { // Update activity check on successful send nsSend.NextActivityCheck = curTicks + 30000; + nsSend.TryShrinkSendBuffer(curTicks); } break; } @@ -625,6 +757,7 @@ public partial class NetState foreach (var ns in Instances) { ns.CheckAlive(curTicks); + ns.TryShrinkSendBuffer(curTicks); } } catch (Exception ex) diff --git a/Projects/Server/Network/NetState/NetState.cs b/Projects/Server/Network/NetState/NetState.cs index e161462fc..af0101ded 100755 --- a/Projects/Server/Network/NetState/NetState.cs +++ b/Projects/Server/Network/NetState/NetState.cs @@ -47,6 +47,20 @@ public partial class NetState : IComparable, IValueLinkListNode _instances = new(2048); public static HashSet Instances => _instances; + // GC's container-aware figure: a heuristic, not a hard bound; fails open when unpopulated + private static bool UnderMemoryCeiling() => + _memoryCeilingPercent <= 0 || + _availableMemoryBytes <= 0 || + Environment.WorkingSet < _availableMemoryBytes / 100 * _memoryCeilingPercent; + + private const long MemoryCeilingWarnIntervalMs = 60000; + private static long _memoryCeilingWarnedAt; + private static bool _memoryCeilingWarned; + + // Reset by MaintainSendBuffers; the transport counts budget refusals + private static int _ceilingRefusals; + private static int _capRefusals; + private readonly string _toString; private ClientVersion _version; private bool _running = true; @@ -59,6 +73,10 @@ public partial class NetState : IComparable, IValueLinkListNode, IValueLinkListNode 0; } + // Grows tier by tier until `needed` fits or a tier is refused. + internal bool TryGrowSendBuffer(int needed) + { + if (_socket == null) + { + return false; + } + + while (_socket.SendBuffer.WritableBytes < needed) + { + if (!TryGrowSendBufferOneTier()) + { + return false; + } + } + + return true; + } + + // One tier; compression learns its output size by retrying + internal bool TryGrowSendBufferOneTier() + { + if (_socket == null) + { + return false; + } + + if (!UnderMemoryCeiling()) + { + _ceilingRefusals++; + + var now = Core.TickCount; + if (!_memoryCeilingWarned || now - (_memoryCeilingWarnedAt + MemoryCeilingWarnIntervalMs) >= 0) + { + _memoryCeilingWarned = true; + _memoryCeilingWarnedAt = now; + logger.Warning("Send buffer growth refused: process is above {Percent}% of available memory", _memoryCeilingPercent); + } + + return false; + } + + if (!_socketManager.TryGrowSendBuffer(_socket)) + { + // At the cap; the transport counts budget refusals + if (_socket.SendBuffer.PhysicalSize >= MaxSendBufferSize) + { + _capRefusals++; + } + + return false; + } + + _sendBufferGrown = true; + _sendBufferGrewAt = Core.TickCount; + logger.Debug("{NetState}: send buffer grown to {Size}", this, _socket.SendBuffer.PhysicalSize); + return true; + } + + // The pool retains the larger buffer + internal bool TryShrinkSendBuffer(long curTicks) + { + if (!_sendBufferGrown || _socket == null || curTicks - (_sendBufferGrewAt + SendBufferHoldMs) < 0) + { + return false; + } + + if (!_socketManager.TryShrinkSendBuffer(_socket)) + { + return false; + } + + _sendBufferGrown = false; + logger.Debug("{NetState}: send buffer returned to {Size}", this, _socket.SendBuffer.PhysicalSize); + return true; + } + public void Send(ReadOnlySpan span) { if (span == ReadOnlySpan.Empty || this.CannotSendPackets()) @@ -499,31 +594,57 @@ public partial class NetState : IComparable, IValueLinkListNode NetworkCompression.DefiniteOverflow) + { + SendBufferExhausted(span.Length); + return; + } + + // Output size is unknown until compressed; grow a tier and retry + while (length <= 0 && TryGrowSendBufferOneTier() && GetSendBuffer(out buffer)) + { + length = NetworkCompression.Compress(span, buffer); + } + + if (length <= 0) + { + SendBufferExhausted(span.Length); + return; + } } } else if (span.Length > buffer.Length) { - SendBufferExhausted(span.Length, buffer.Length); - return; + if (!TryGrowSendBuffer(span.Length) || !GetSendBuffer(out buffer)) + { + SendBufferExhausted(span.Length); + return; + } + + span.CopyTo(buffer); } else { @@ -558,9 +679,9 @@ public partial class NetState : IComparable, IValueLinkListNode /// /// High unacked means a slow client holding the buffer; needed approaching capacity means the - /// buffer is too small for this shard and network.sendBufferSize should be raised. + /// buffer is too small for this shard and network.sendBufferMaxSize should be raised. /// - private void SendBufferExhausted(int needed, int writable) + private void SendBufferExhausted(int needed) { // One report per disconnect; the first reason wins if (_disconnectQueued) @@ -568,12 +689,14 @@ public partial class NetState : IComparable, IValueLinkListNode - + diff --git a/dev-docs/server-requirements.md b/dev-docs/server-requirements.md index 0b2c50569..db9c0907b 100644 --- a/dev-docs/server-requirements.md +++ b/dev-docs/server-requirements.md @@ -68,8 +68,13 @@ Optional systems can add substantially more. The pathfinding prebake (`pathfinding.prebakeMaps`) peaks above 1 GB of heap while baking. Budget for it or leave it off on small hosts. -Network buffers are minor by comparison: 64 KB receive plus a configurable 256 KB send -(`network.sendBufferSize`) per connection, so 100 players is roughly 32 MB. +Network buffers are 64 KB receive plus a configurable send buffer per connection. Send memory is +`network.sendBufferSize` at rest and can grow to `network.sendBufferMaxSize` under load. Shared +send-buffer tier memory is capped by `network.sendBufferGrowthBudget`, and growth is refused when +process memory exceeds `network.memoryCeilingPercent` of available memory. The worst case is the +base send-buffer size times the connection count, plus the shared growth budget: at the defaults, +100 players is roughly 32 MB at rest, and the growth budget can add up to another 256 MB under +load. ModernUO runs **Workstation GC**, which is the right default for small hosts. Do not switch to Server GC on a 2-core box. @@ -110,6 +115,9 @@ See the README for the full supported list. Two things are worth calling out: | `world.useMultithreadedSaves` | `true` | Set `false` on 2-core hosts so saves do not contend with the game loop. | | `pathfinding.prebakeMaps` | varies | Leave off on memory-constrained hosts; it peaks above 1 GB while baking. | | `network.sendBufferSize` | 256 KB | Lower it if you are memory-bound with many connections. | +| `network.sendBufferMaxSize` | 2 MB (`2097152`) | Ceiling a single connection's send buffer can grow to under load. Lower it on memory-constrained hosts; raise it if slow clients are disconnected with "send buffer exhausted". | +| `network.sendBufferGrowthBudget` | 256 MB (`268435456`) | Cap on the shared memory the larger send-buffer tiers may use. Lower it on memory-constrained hosts. | +| `network.memoryCeilingPercent` | 80% | Refuse send-buffer growth once the process is above this share of available memory; 0 turns the check off. | | `autoArchive.*` retention | 24h/30d/12m | Reduce if disk is tight. | ## Am I undersized?