diff --git a/Projects/Server.Tests/Tests/Network/NetStateDisconnectTests.cs b/Projects/Server.Tests/Tests/Network/NetStateDisconnectTests.cs index adc513e2c..d1f273db3 100644 --- a/Projects/Server.Tests/Tests/Network/NetStateDisconnectTests.cs +++ b/Projects/Server.Tests/Tests/Network/NetStateDisconnectTests.cs @@ -225,12 +225,10 @@ public class NetStateDisconnectTests try { - var capacity = ns._socket.SendBuffer.PhysicalSize; + ns.Send(new byte[NetState.MaxSendBufferSize + 1]); // cannot fit; queues the disconnect + ns.Send(new byte[NetState.MaxSendBufferSize + 2]); // same tick; must not re-report - ns.Send(new byte[capacity + 1]); // cannot fit; queues the disconnect - ns.Send(new byte[capacity + 2]); // same tick; must not re-report - - Assert.Contains($"needed {capacity + 1}", ns._disconnectReason); + Assert.Contains($"needed {NetState.MaxSendBufferSize + 1}", ns._disconnectReason); } finally { diff --git a/Projects/Server.Tests/Tests/Network/NetStateSendBufferTests.cs b/Projects/Server.Tests/Tests/Network/NetStateSendBufferTests.cs new file mode 100644 index 000000000..2199052d2 --- /dev/null +++ b/Projects/Server.Tests/Tests/Network/NetStateSendBufferTests.cs @@ -0,0 +1,451 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net.Sockets; +using System.Network; +using System.Threading; +using Server.Network; +using Xunit; + +namespace Server.Tests.Network; + +[Collection("Sequential Server Tests")] +public class NetStateSendBufferTests +{ + private static NetState CreateAuthenticatedNetState(out Socket client) + { + var ns = PacketTestUtilities.CreateTestNetState(out client); + ns.Account = new MockAccount(); + return ns; + } + + private static byte[] Pattern(int length, int seed) + { + var data = new byte[length]; + new System.Random(seed).NextBytes(data); + return data; + } + + // A random prefix lands under the target; zeros (2 bits each) walk it up a byte at a time + private static byte[] CompressesToExactly(int compressedLength, int seed) + { + var randomLength = compressedLength * 5 / 8; + var data = new byte[compressedLength * 8]; + Pattern(randomLength, seed).CopyTo(data, 0); + + var scratch = new byte[compressedLength * 2 + 16]; + for (var length = randomLength; length < data.Length; length++) + { + var compressed = NetworkCompression.Compress(data.AsSpan(0, length), scratch); + Assert.InRange(compressed, 1, compressedLength); + + if (compressed == compressedLength) + { + return data[..length]; + } + } + + throw new InvalidOperationException($"No chunk compresses to exactly {compressedLength} bytes"); + } + + private static byte[] ReadAll(Socket client, int length) + { + var received = new byte[length]; + var total = 0; + var deadline = Stopwatch.StartNew(); + while (total < length && deadline.ElapsedMilliseconds < 10000) + { + NetState.Slice(); + + // Poll takes microseconds, not milliseconds + if (client.Poll(50_000, SelectMode.SelectRead)) + { + var read = client.Receive(received, total, length - total, SocketFlags.None); + Assert.NotEqual(0, read); + total += read; + } + } + + Assert.Equal(length, total); + return received; + } + + // Peer receipt does not free the buffer; the completion must land first + private static void WaitForDrain(NetState ns) + { + var deadline = Stopwatch.StartNew(); + while (deadline.ElapsedMilliseconds < 10000) + { + NetState.Slice(); + + if (ns._socket.SendBuffer.ReadableBytes == 0 && ns._socket.SendBuffer.InFlightBytes == 0) + { + break; + } + + Thread.Sleep(5); + } + + Assert.Equal(0, ns._socket.SendBuffer.ReadableBytes); + Assert.Equal(0, ns._socket.SendBuffer.InFlightBytes); + } + + [Fact] + public void Send_GrowsInsteadOfDisconnecting_AndStreamStaysIntact() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + // No Slice(): nothing drains, so the base buffer overflows + var chunk = baseSize / 4; + var expected = new byte[chunk * 6]; + for (var i = 0; i < 6; i++) + { + var data = Pattern(chunk, i); + data.CopyTo(expected, i * chunk); + ns.Send(data); + } + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.True(ns._socket.SendBuffer.PhysicalSize > baseSize); + Assert.True(ns._sendBufferGrown); + + Assert.Equal(expected, ReadAll(client, expected.Length)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_CompressedGrowth_RetriesAndStreamStaysIntact() + { + var ns = CreateAuthenticatedNetState(out var client); + ns.CompressionEnabled = true; + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + // No Slice(): compressed output accumulates until the base overflows + var chunkLength = baseSize / 4; + var expected = new byte[(chunkLength * 2 + 4) * 6]; + var scratch = new byte[chunkLength * 2 + 4]; + var expectedLength = 0; + for (var i = 0; i < 6; i++) + { + var data = Pattern(chunkLength, i + 100); + var compressedLength = NetworkCompression.Compress(data, scratch); + Assert.True(compressedLength > 0); + scratch.AsSpan(0, compressedLength).CopyTo(expected.AsSpan(expectedLength)); + expectedLength += compressedLength; + ns.Send(data); + } + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.True(ns._socket.SendBuffer.PhysicalSize > baseSize); + Assert.True(ns._sendBufferGrown); + + Array.Resize(ref expected, expectedLength); + Assert.Equal(expected, ReadAll(client, expected.Length)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_CompressedGrowth_FullBuffer_GrowsOneTierAndFits() + { + var ns = CreateAuthenticatedNetState(out var client); + ns.CompressionEnabled = true; + var baseSize = ns._socket.SendBuffer.PhysicalSize; + var expected = new List(baseSize * 2); + + void SendAndRecord(byte[] data) + { + var scratch = new byte[data.Length * 2 + 4]; + var compressedLength = NetworkCompression.Compress(data, scratch); + Assert.True(compressedLength > 0); + expected.AddRange(scratch.AsSpan(0, compressedLength)); + ns.Send(data); + } + + try + { + // Chunks sized so the fill cannot overflow (random bytes never compress; ratio at most 11/8) + var seed = 600; + while (ns._socket.SendBuffer.WritableBytes > 4096) + { + var writable = ns._socket.SendBuffer.WritableBytes; + SendAndRecord(Pattern(Math.Min(baseSize / 8, (writable - 2048) * 8 / 11), seed++)); + } + + // Full; no span to compress into + SendAndRecord(CompressesToExactly(ns._socket.SendBuffer.WritableBytes, seed)); + Assert.Equal(0, ns._socket.SendBuffer.WritableBytes); + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + + // Raw exceeds the remainder; compressed fits in one tier + SendAndRecord(new byte[Math.Min(baseSize * 3 / 4, NetworkCompression.DefiniteOverflow)]); + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.Equal(baseSize * 2, ns._socket.SendBuffer.PhysicalSize); + Assert.True(ns._sendBufferGrown); + + Assert.Equal(expected.ToArray(), ReadAll(client, expected.Count)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_WithPartialRemainder_GrowsAndCopiesFullPayload() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + var first = Pattern(baseSize / 2, 200); + var second = Pattern(baseSize / 2 + 1, 201); + var expected = new byte[first.Length + second.Length]; + first.CopyTo(expected, 0); + second.CopyTo(expected, first.Length); + + // Remainder too small for the second send + ns.Send(first); + ns.Send(second); + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.True(ns._socket.SendBuffer.PhysicalSize > baseSize); + Assert.True(ns._sendBufferGrown); + + Assert.Equal(expected, ReadAll(client, expected.Length)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_WithSendInFlight_GrowsAndStreamStaysIntact() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + // A small peer buffer keeps the first send in flight through the growth + client.ReceiveBufferSize = 4096; + + var first = Pattern(baseSize / 2, 300); + var second = Pattern(baseSize / 2 + 1, 301); + + ns.Send(first); + NetState.Slice(); // posts the first send + Assert.True(ns._socket.SendBuffer.InFlightBytes > 0); + + ns.Send(second); + + Assert.True(ns.Running); + Assert.Equal(string.Empty, ns._disconnectReason); + Assert.True(ns._socket.SendBuffer.PhysicalSize > baseSize); + Assert.True(ns._sendBufferGrown); + + var expected = new byte[first.Length + second.Length]; + first.CopyTo(expected, 0); + second.CopyTo(expected, first.Length); + + Assert.Equal(expected, ReadAll(client, expected.Length)); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Shrink_AfterDrainAndHold_ReturnsToBase() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + var chunk = baseSize / 4; + for (var i = 0; i < 6; i++) + { + ns.Send(Pattern(chunk, i)); + } + + Assert.True(ns._sendBufferGrown); + ReadAll(client, chunk * 6); + WaitForDrain(ns); + + var grewAt = ns._sendBufferGrewAt; + Assert.False(ns.TryShrinkSendBuffer(grewAt + NetState.SendBufferHoldMs - 1)); + Assert.True(ns.TryShrinkSendBuffer(grewAt + NetState.SendBufferHoldMs)); + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + Assert.False(ns._sendBufferGrown); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Shrink_ThroughAliveSweep_ReturnsToBase() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + var previousTicks = Core._tickCount; + + try + { + var chunk = baseSize / 4; + for (var i = 0; i < 6; i++) + { + ns.Send(Pattern(chunk, i + 400)); + } + + Assert.True(ns._sendBufferGrown); + ReadAll(client, chunk * 6); + WaitForDrain(ns); + + // The sweep shrinks; nothing here calls TryShrinkSendBuffer + Core._tickCount = ns._sendBufferGrewAt + NetState.SendBufferHoldMs; + NetState.CheckAllAlive(); + + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + Assert.False(ns._sendBufferGrown); + } + finally + { + Core._tickCount = previousTicks; + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_PastTheMaximum_FallsBackToExhaustion() + { + var ns = CreateAuthenticatedNetState(out var client); + + try + { + var chunk = 64 * 1024; + var writes = NetState.MaxSendBufferSize / chunk + 1; + for (var i = 0; i < writes; i++) + { + ns.Send(Pattern(chunk, i)); + } + + Assert.Contains("Send buffer exhausted", ns._disconnectReason); + Assert.Equal(NetState.MaxSendBufferSize, ns._socket.SendBuffer.PhysicalSize); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Fact] + public void Send_AboveMemoryCeiling_DoesNotGrow() + { + var previous = NetState._availableMemoryBytes; + NetState ns = null; + Socket client = null; + + try + { + ns = CreateAuthenticatedNetState(out client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + NetState._availableMemoryBytes = 1; // any working set is above 80% of one byte + + for (var i = 0; i < 6; i++) + { + ns.Send(Pattern(baseSize / 4, i)); + } + + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + Assert.Contains("Send buffer exhausted", ns._disconnectReason); + } + finally + { + NetState._availableMemoryBytes = previous; + ns?.Dispose(); + client?.Close(); + } + } + + [Fact] + public void Send_OnClosingSocket_DoesNotGrow() + { + var ns = CreateAuthenticatedNetState(out var client); + var baseSize = ns._socket.SendBuffer.PhysicalSize; + + try + { + ns.Disconnect("test"); + NetState.Slice(); // handoff + + // CannotSendPackets short-circuits before any growth + ns.Send(Pattern(baseSize * 2, 500)); + + Assert.Equal(baseSize, ns._socket.SendBuffer.PhysicalSize); + Assert.False(ns._sendBufferGrown); + } + finally + { + ns.Dispose(); + client.Close(); + } + } + + [Theory] + [InlineData(3 * 1024 * 1024, 4 * 1024 * 1024)] // not a power of two; rounded up + [InlineData(512 * 1024 * 1024, 256 * 1024 * 1024)] // above the transport ceiling; capped + [InlineData(1024, 64 * 1024)] // below the minimum; raised + [InlineData(2 * 1024 * 1024, 2 * 1024 * 1024)] // already valid; untouched + public void CoercePowerOfTwoSetting_CoercesToATierTheTransportAccepts(int configured, int expected) => + Assert.Equal(expected, NetState.CoercePowerOfTwoSetting("network.sendBufferMaxSize", configured, 64 * 1024)); + + [Fact] + public void CoerceSendBufferGrowthBudget_ClampsNegativeAndRaisesBelowOneSlab() + { + const int sendBufferSize = 256 * 1024; + const int maxSendBufferSize = 2 * 1024 * 1024; + var minimum = RingSocketManager.MinimumSendBufferGrowthBudget(sendBufferSize); + + Assert.Equal(0L, NetState.CoerceSendBufferGrowthBudget(-1, sendBufferSize, maxSendBufferSize)); + + // zero means never grow + Assert.Equal(0L, NetState.CoerceSendBufferGrowthBudget(0, sendBufferSize, maxSendBufferSize)); + + Assert.Equal(minimum, NetState.CoerceSendBufferGrowthBudget(1, sendBufferSize, maxSendBufferSize)); + Assert.Equal(minimum * 4, NetState.CoerceSendBufferGrowthBudget(minimum * 4, sendBufferSize, maxSendBufferSize)); + + // growth off; budget untouched + Assert.Equal(1L, NetState.CoerceSendBufferGrowthBudget(1, sendBufferSize, sendBufferSize)); + } +} diff --git a/Projects/Server/Network/NetState/DumpNetStates.cs b/Projects/Server/Network/NetState/DumpNetStates.cs index 018ebb7e1..1dae56203 100644 --- a/Projects/Server/Network/NetState/DumpNetStates.cs +++ b/Projects/Server/Network/NetState/DumpNetStates.cs @@ -28,11 +28,11 @@ public static class DumpNetStates { using var file = new StreamWriter($"netstatedump-{Core.Now:yyyy-M-d-HH-mm-ss}_{Core.TickCount}.csv"); - file.WriteLine("NetState, ConnectedOn, NextActivityCheck, SocketConnected, ProtocolState, ParserState"); + file.WriteLine("NetState, ConnectedOn, NextActivityCheck, SocketConnected, ProtocolState, ParserState, SendBufferSize"); foreach (var ns in NetState.Instances) { - file.WriteLine($"{ns}, {ns.ConnectedOn}, {ns.NextActivityCheck}, {ns.IsConnected}, {ns._protocolState}, {ns._parserState}"); + file.WriteLine($"{ns}, {ns.ConnectedOn}, {ns.NextActivityCheck}, {ns.IsConnected}, {ns._protocolState}, {ns._parserState}, {ns._socket?.SendBuffer.PhysicalSize ?? 0}"); } } } diff --git a/Projects/Server/Network/NetState/NetState.Network.cs b/Projects/Server/Network/NetState/NetState.Network.cs index 5bedb504c..e393446b1 100644 --- a/Projects/Server/Network/NetState/NetState.Network.cs +++ b/Projects/Server/Network/NetState/NetState.Network.cs @@ -32,8 +32,26 @@ public partial class NetState private const int RecvBufferSize = 1024 * 64; // 64KB recv buffers private const int DefaultSendBufferSize = 1024 * 256; // 256KB send buffers private const int MinSendBufferSize = 1024 * 64; // Platform allocation granularity + private const int DefaultMaxSendBufferSize = 1024 * 1024 * 2; // 2 MB + private const long DefaultSendBufferGrowthBudget = 1024L * 1024 * 256; // 256 MB + private const int DefaultMemoryCeilingPercent = 80; + + // Transport ceiling; larger values overflow its tier enumeration + private const int TransportMaxSendBufferSize = 1024 * 1024 * 256; // 256 MB private const int MaxConnections = 4096; // Max concurrent connections + internal static int MaxSendBufferSize { get; private set; } + private static long _sendBufferGrowthBudget; + private static int _memoryCeilingPercent; + + // Refreshed by the maintenance sweep; internal for tests + internal static long _availableMemoryBytes; + + private static Timer.DelayCallTimer _maintenanceTimer; + private static long _lastTierCapacityBytes; + private static int _lastTierInUse; + private static int _lastTierRetainFloor; + private static readonly Queue _disposed = []; private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds @@ -121,16 +139,26 @@ public partial class NetState // latency is roughly completion RTT / this value. var maxOutstandingSends = ServerConfiguration.GetOrUpdateSetting("network.maxOutstandingSends", 32); - // Initialize IORingGroup - var ring = IORingGroup.Create( - queueSize: MaxConnections * 2, - maxConnections: MaxConnections, - maxOutstandingSends: maxOutstandingSends - ); - // Per-connection send buffer: the lever for "send buffer exhausted" disconnects, and the // per-connection memory ceiling. var sendBufferSize = GetSendBufferSize(); + MaxSendBufferSize = GetPowerOfTwoSetting("network.sendBufferMaxSize", DefaultMaxSendBufferSize, sendBufferSize); + _sendBufferGrowthBudget = CoerceSendBufferGrowthBudget( + ServerConfiguration.GetOrUpdateSetting("network.sendBufferGrowthBudget", DefaultSendBufferGrowthBudget), + sendBufferSize, + MaxSendBufferSize + ); + // 0 disables the ceiling + _memoryCeilingPercent = Math.Clamp(ServerConfiguration.GetOrUpdateSetting("network.memoryCeilingPercent", DefaultMemoryCeilingPercent), 0, 100); + _availableMemoryBytes = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + + const int maxBufferSlabs = 32; + var ring = IORingGroup.Create( + queueSize: MaxConnections * 2, + maxConnections: MaxConnections, + maxOutstandingSends: maxOutstandingSends, + maxRegisteredBuffers: RingSocketManager.RequiredRegisteredBuffers(MaxConnections, sendBufferSize, MaxSendBufferSize, _sendBufferGrowthBudget, maxBufferSlabs) + ); // Create socket manager which handles buffer pools and socket lifecycle _socketManager = new RingSocketManager( @@ -139,8 +167,51 @@ public partial class NetState recvBufferSize: RecvBufferSize, sendBufferSize: sendBufferSize, initialBufferSlabs: 8, - maxBufferSlabs: 32 + maxBufferSlabs: maxBufferSlabs, + maxSendBufferSize: MaxSendBufferSize, + sendBufferGrowthBudget: _sendBufferGrowthBudget ); + + _maintenanceTimer = Timer.DelayCall(TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), MaintainSendBuffers); + } + + internal static void MaintainSendBuffers() + { + if (_socketManager == null) + { + return; + } + + // Container limits can change + _availableMemoryBytes = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + + var ceilingRefusals = _ceilingRefusals; + var capRefusals = _capRefusals; + _ceilingRefusals = 0; + _capRefusals = 0; + + var stats = _socketManager.Maintain(); + var changed = stats.TierCapacityBytes != _lastTierCapacityBytes || + stats.TierInUse != _lastTierInUse || + stats.TierRetainFloor != _lastTierRetainFloor; + _lastTierCapacityBytes = stats.TierCapacityBytes; + _lastTierInUse = stats.TierInUse; + _lastTierRetainFloor = stats.TierRetainFloor; + + // Quiet unless something moved + if (changed || stats.BuffersReleased > 0 || stats.GrowthRefusals > 0 || capRefusals > 0 || ceilingRefusals > 0) + { + logger.Debug( + "Send buffer tiers: {Capacity} bytes of tier capacity, {InUse} buffers in use, floor {Floor} buffers, released {Released}, refused: budget {BudgetRefusals}, at max {CapRefusals}, ceiling {CeilingRefusals}", + stats.TierCapacityBytes, + stats.TierInUse, + stats.TierRetainFloor, + stats.BuffersReleased, + stats.GrowthRefusals, + capRefusals, + ceilingRefusals + ); + } } /// @@ -148,29 +219,89 @@ public partial class NetState /// allocation granularity. IORingBuffer requires this and would otherwise throw at socket /// creation rather than at startup. /// - private static int GetSendBufferSize() + private static int GetSendBufferSize() => + GetPowerOfTwoSetting("network.sendBufferSize", DefaultSendBufferSize, MinSendBufferSize); + + private static int GetPowerOfTwoSetting(string key, int defaultValue, int minimum) => + CoercePowerOfTwoSetting(key, ServerConfiguration.GetOrUpdateSetting(key, defaultValue), minimum); + + /// + /// Clamps to a power of two between and the transport ceiling. + /// + internal static int CoercePowerOfTwoSetting(string key, int configured, int minimum) { - var configured = ServerConfiguration.GetOrUpdateSetting("network.sendBufferSize", DefaultSendBufferSize); - var size = Math.Max(MinSendBufferSize, configured); + var size = configured; + + if (size > TransportMaxSendBufferSize) + { + logger.Warning( + "{Key} {Configured} is above the transport maximum {Maximum} (capped); using {Adjusted}", + key, + configured, + TransportMaxSendBufferSize, + TransportMaxSendBufferSize + ); + + size = TransportMaxSendBufferSize; + } + + if (size < minimum) + { + logger.Warning( + "{Key} {Configured} is below the minimum {Minimum} (raised); using {Adjusted}", + key, + configured, + minimum, + minimum + ); + + size = minimum; + } if (!BitOperations.IsPow2(size)) { - size = (int)BitOperations.RoundUpToPowerOf2((uint)size); - } + // Rounding up cannot cross the ceiling, itself a power of two + var rounded = (int)BitOperations.RoundUpToPowerOf2((uint)size); - if (size != configured) - { - logger.Warning( - "network.sendBufferSize {Configured} is not a power of two of at least {Minimum}; using {Adjusted}", - configured, - MinSendBufferSize, - size - ); + logger.Warning("{Key} {Configured} is not a power of two; using {Adjusted}", key, configured, rounded); + size = rounded; } return size; } + /// + /// Negative disables growth; below one tier slab is raised to the minimum. + /// + internal static long CoerceSendBufferGrowthBudget(long configured, int sendBufferSize, int maxSendBufferSize) + { + if (configured < 0) + { + logger.Warning("network.sendBufferGrowthBudget {Configured} is negative; using 0 (growth disabled)", configured); + return 0; + } + + if (configured == 0 || maxSendBufferSize <= sendBufferSize) + { + return configured; + } + + // Tier buffers are allocated a slab at a time. + var minimumBudget = RingSocketManager.MinimumSendBufferGrowthBudget(sendBufferSize); + if (configured < minimumBudget) + { + logger.Warning( + "network.sendBufferGrowthBudget {Configured} is below one tier slab; using {Minimum}", + configured, + minimumBudget + ); + + return minimumBudget; + } + + return configured; + } + /// /// Starts the network server on configured listening addresses. /// @@ -494,6 +625,7 @@ public partial class NetState { // Update activity check on successful send nsSend.NextActivityCheck = curTicks + 30000; + nsSend.TryShrinkSendBuffer(curTicks); } break; } @@ -625,6 +757,7 @@ public partial class NetState foreach (var ns in Instances) { ns.CheckAlive(curTicks); + ns.TryShrinkSendBuffer(curTicks); } } catch (Exception ex) diff --git a/Projects/Server/Network/NetState/NetState.cs b/Projects/Server/Network/NetState/NetState.cs index e161462fc..af0101ded 100755 --- a/Projects/Server/Network/NetState/NetState.cs +++ b/Projects/Server/Network/NetState/NetState.cs @@ -47,6 +47,20 @@ public partial class NetState : IComparable, IValueLinkListNode _instances = new(2048); public static HashSet Instances => _instances; + // GC's container-aware figure: a heuristic, not a hard bound; fails open when unpopulated + private static bool UnderMemoryCeiling() => + _memoryCeilingPercent <= 0 || + _availableMemoryBytes <= 0 || + Environment.WorkingSet < _availableMemoryBytes / 100 * _memoryCeilingPercent; + + private const long MemoryCeilingWarnIntervalMs = 60000; + private static long _memoryCeilingWarnedAt; + private static bool _memoryCeilingWarned; + + // Reset by MaintainSendBuffers; the transport counts budget refusals + private static int _ceilingRefusals; + private static int _capRefusals; + private readonly string _toString; private ClientVersion _version; private bool _running = true; @@ -59,6 +73,10 @@ public partial class NetState : IComparable, IValueLinkListNode, IValueLinkListNode 0; } + // Grows tier by tier until `needed` fits or a tier is refused. + internal bool TryGrowSendBuffer(int needed) + { + if (_socket == null) + { + return false; + } + + while (_socket.SendBuffer.WritableBytes < needed) + { + if (!TryGrowSendBufferOneTier()) + { + return false; + } + } + + return true; + } + + // One tier; compression learns its output size by retrying + internal bool TryGrowSendBufferOneTier() + { + if (_socket == null) + { + return false; + } + + if (!UnderMemoryCeiling()) + { + _ceilingRefusals++; + + var now = Core.TickCount; + if (!_memoryCeilingWarned || now - (_memoryCeilingWarnedAt + MemoryCeilingWarnIntervalMs) >= 0) + { + _memoryCeilingWarned = true; + _memoryCeilingWarnedAt = now; + logger.Warning("Send buffer growth refused: process is above {Percent}% of available memory", _memoryCeilingPercent); + } + + return false; + } + + if (!_socketManager.TryGrowSendBuffer(_socket)) + { + // At the cap; the transport counts budget refusals + if (_socket.SendBuffer.PhysicalSize >= MaxSendBufferSize) + { + _capRefusals++; + } + + return false; + } + + _sendBufferGrown = true; + _sendBufferGrewAt = Core.TickCount; + logger.Debug("{NetState}: send buffer grown to {Size}", this, _socket.SendBuffer.PhysicalSize); + return true; + } + + // The pool retains the larger buffer + internal bool TryShrinkSendBuffer(long curTicks) + { + if (!_sendBufferGrown || _socket == null || curTicks - (_sendBufferGrewAt + SendBufferHoldMs) < 0) + { + return false; + } + + if (!_socketManager.TryShrinkSendBuffer(_socket)) + { + return false; + } + + _sendBufferGrown = false; + logger.Debug("{NetState}: send buffer returned to {Size}", this, _socket.SendBuffer.PhysicalSize); + return true; + } + public void Send(ReadOnlySpan span) { if (span == ReadOnlySpan.Empty || this.CannotSendPackets()) @@ -499,31 +594,57 @@ public partial class NetState : IComparable, IValueLinkListNode NetworkCompression.DefiniteOverflow) + { + SendBufferExhausted(span.Length); + return; + } + + // Output size is unknown until compressed; grow a tier and retry + while (length <= 0 && TryGrowSendBufferOneTier() && GetSendBuffer(out buffer)) + { + length = NetworkCompression.Compress(span, buffer); + } + + if (length <= 0) + { + SendBufferExhausted(span.Length); + return; + } } } else if (span.Length > buffer.Length) { - SendBufferExhausted(span.Length, buffer.Length); - return; + if (!TryGrowSendBuffer(span.Length) || !GetSendBuffer(out buffer)) + { + SendBufferExhausted(span.Length); + return; + } + + span.CopyTo(buffer); } else { @@ -558,9 +679,9 @@ public partial class NetState : IComparable, IValueLinkListNode /// /// High unacked means a slow client holding the buffer; needed approaching capacity means the - /// buffer is too small for this shard and network.sendBufferSize should be raised. + /// buffer is too small for this shard and network.sendBufferMaxSize should be raised. /// - private void SendBufferExhausted(int needed, int writable) + private void SendBufferExhausted(int needed) { // One report per disconnect; the first reason wins if (_disconnectQueued) @@ -568,12 +689,14 @@ public partial class NetState : IComparable, IValueLinkListNode - + diff --git a/dev-docs/server-requirements.md b/dev-docs/server-requirements.md index 0b2c50569..db9c0907b 100644 --- a/dev-docs/server-requirements.md +++ b/dev-docs/server-requirements.md @@ -68,8 +68,13 @@ Optional systems can add substantially more. The pathfinding prebake (`pathfinding.prebakeMaps`) peaks above 1 GB of heap while baking. Budget for it or leave it off on small hosts. -Network buffers are minor by comparison: 64 KB receive plus a configurable 256 KB send -(`network.sendBufferSize`) per connection, so 100 players is roughly 32 MB. +Network buffers are 64 KB receive plus a configurable send buffer per connection. Send memory is +`network.sendBufferSize` at rest and can grow to `network.sendBufferMaxSize` under load. Shared +send-buffer tier memory is capped by `network.sendBufferGrowthBudget`, and growth is refused when +process memory exceeds `network.memoryCeilingPercent` of available memory. The worst case is the +base send-buffer size times the connection count, plus the shared growth budget: at the defaults, +100 players is roughly 32 MB at rest, and the growth budget can add up to another 256 MB under +load. ModernUO runs **Workstation GC**, which is the right default for small hosts. Do not switch to Server GC on a 2-core box. @@ -110,6 +115,9 @@ See the README for the full supported list. Two things are worth calling out: | `world.useMultithreadedSaves` | `true` | Set `false` on 2-core hosts so saves do not contend with the game loop. | | `pathfinding.prebakeMaps` | varies | Leave off on memory-constrained hosts; it peaks above 1 GB while baking. | | `network.sendBufferSize` | 256 KB | Lower it if you are memory-bound with many connections. | +| `network.sendBufferMaxSize` | 2 MB (`2097152`) | Ceiling a single connection's send buffer can grow to under load. Lower it on memory-constrained hosts; raise it if slow clients are disconnected with "send buffer exhausted". | +| `network.sendBufferGrowthBudget` | 256 MB (`268435456`) | Cap on the shared memory the larger send-buffer tiers may use. Lower it on memory-constrained hosts. | +| `network.memoryCeilingPercent` | 80% | Refuse send-buffer growth once the process is above this share of available memory; 0 turns the check off. | | `autoArchive.*` retention | 24h/30d/12m | Reduce if disk is tight. | ## Am I undersized?