fix(core): Tightens the network stack (#479)

- [X] Removes network pause/resume
- [X] Adds back packet profiler
- [X] Adds state machine to keep track and trace netstates
- [X] Changes NetState.Running back to using an interlock exchange
- [X] Adds preliminary packet throttling support


### Packet Throttling
- `[GetThrottle <packetId>` to get the delay in milliseconds for that packet
- `[SetThrottle <packetId> <delay>` to set the delay in milliseconds for that packet
The settings are saved to `Configuration/throttles.json`
This commit is contained in:
Kamron Batman 2021-02-07 23:30:56 -08:00 committed by GitHub
parent 1736469ac0
commit 8903028b5f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 1018 additions and 405 deletions

View file

@ -23,6 +23,7 @@ using System.Net.Sockets;
using System.Runtime.CompilerServices;
using System.Threading;
using Server.Accounting;
using Server.Diagnostics;
using Server.Gumps;
using Server.HuePickers;
using Server.Items;
@ -37,57 +38,102 @@ namespace Server.Network
public partial class NetState : IComparable<NetState>
{
private static int RecvPipeSize = 1024 * 64 + 1;
private static int SendPipeSize = 1024 * 256 + 1;
private const int RecvPipeSize = 1024 * 64;
private const int SendPipeSize = 1024 * 256;
private static int GumpCap = 512;
private static int HuePickerCap = 512;
private static int MenuCap = 512;
private static int PacketPerSecondThreshold = 3000;
private static readonly Queue<NetState> FlushPending = new(2048);
private static readonly ConcurrentQueue<NetState> Disposed = new();
private static NetworkState NetworkState = NetworkState.ResumeState;
public static NetStateCreatedCallback CreatedCallback { get; set; }
private readonly string _toString;
private ClientVersion _version;
private long _nextActivityCheck;
private volatile bool _running;
private int _running;
private volatile DecodePacket _packetDecoder;
private volatile EncodePacket _packetEncoder;
private bool _flushQueued;
private readonly long[] _packetThrottles = new long[0x100];
private readonly long[] _packetCounts = new long[0x100];
private string _disconnectReason = string.Empty;
internal int _authId;
internal int _seed;
internal ParserState _parserState = ParserState.Uninitialized;
internal ProtocolState _protocolState = ProtocolState.Uninitialized;
internal RecvState _recvState = RecvState.Uninitialized;
internal SendState _sendState = SendState.Uninitialized;
internal enum ParserState
{
Uninitialized,
AwaitingNextPacket,
AwaitingPartialPacket,
ProcessingPacket,
Throttled,
Error
}
internal enum ProtocolState
{
Uninitialized,
AwaitingSeed, // Based on the way the seed arrives, we know if this is a login server or a game server connection
LoginServer_AwaitingLogin,
LoginServer_AwaitingServerSelect,
LoginServer_ServerSelectAck,
GameServer_AwaitingGameServerLogin,
GameServer_LoggedIn,
Error
}
internal enum RecvState
{
Uninitialized,
AwaitingMemory,
AwaitingRecv,
DataReceived,
Exited,
}
internal enum SendState
{
Uninitialized,
AwaitingData,
Sending,
SendCompleted,
Exited,
}
public static void Configure()
{
RecvPipeSize = ServerConfiguration.GetOrUpdateSetting("netstate.recvPipeSize", RecvPipeSize);
SendPipeSize = ServerConfiguration.GetOrUpdateSetting("netstate.sendPipeSize", SendPipeSize);
GumpCap = ServerConfiguration.GetOrUpdateSetting("netstate.gumpCap", GumpCap);
HuePickerCap = ServerConfiguration.GetOrUpdateSetting("netstate.huePickerCap", HuePickerCap);
MenuCap = ServerConfiguration.GetOrUpdateSetting("netstate.menuCap", MenuCap);
PacketPerSecondThreshold = ServerConfiguration.GetOrUpdateSetting("netstate.packetsPerSecondThreshold", PacketPerSecondThreshold);
}
public static void Initialize()
{
var checkAliveDuration = TimeSpan.FromMinutes(1.5);
Timer.DelayCall(checkAliveDuration, checkAliveDuration, CheckAllAlive);
Timer.DelayCall(TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1.5), CheckAllAlive);
}
public NetState(ISocket connection)
{
_running = false;
Connection = connection;
Seeded = false;
Gumps = new List<Gump>();
HuePickers = new List<HuePicker>();
Menus = new List<IMenu>();
Trades = new List<SecureTrade>();
var recvBuffer = GC.AllocateUninitializedArray<byte>(RecvPipeSize);
RecvPipe = new Pipe<byte>(recvBuffer);
var sendBuffer = GC.AllocateUninitializedArray<byte>(SendPipeSize);
SendPipe = new Pipe<byte>(sendBuffer);
RecvPipe = new Pipe<byte>(GC.AllocateUninitializedArray<byte>(RecvPipeSize));
SendPipe = new Pipe<byte>(GC.AllocateUninitializedArray<byte>(SendPipeSize));
_nextActivityCheck = Core.TickCount + 30000;
try
@ -111,8 +157,6 @@ namespace Server.Network
public TimeSpan ConnectedFor => DateTime.UtcNow - ConnectedOn;
public DateTime ThrottledUntil { get; set; }
public IPAddress Address { get; }
public DecodePacket PacketDecoder
@ -133,7 +177,11 @@ namespace Server.Network
public List<SecureTrade> Trades { get; }
public bool Running => _running;
public bool Running
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _running == 1;
}
public bool Seeded { get; set; }
@ -163,6 +211,52 @@ namespace Server.Network
public int CompareTo(NetState other) => string.CompareOrdinal(_toString, other?._toString);
private void SetPacketTime(int packetID)
{
if (packetID < 0 || packetID >= 0x100)
{
return;
}
_packetThrottles[packetID] = Core.TickCount;
}
public long GetPacketDelay(int packetID)
{
if (packetID < 0 || packetID >= 0x100)
{
return 0;
}
return _packetThrottles[packetID];
}
private void UpdatePacketCount(int packetID)
{
if (packetID < 0 || packetID >= 0x100)
{
return;
}
_packetCounts[packetID]++;
}
public int CheckPacketCounts()
{
for (int i = 0; i < _packetCounts.Length; i++)
{
long count = _packetCounts[i];
_packetCounts[i] = 0;
if (count > PacketPerSecondThreshold)
{
return i;
}
}
return 0;
}
public void ValidateAllTrades()
{
for (var i = Trades.Count - 1; i >= 0; --i)
@ -270,7 +364,7 @@ namespace Server.Network
else
{
WriteConsole("Exceeded menu cap, disconnecting...");
Disconnect();
Disconnect("Exceeded menu cap.");
}
}
@ -300,7 +394,7 @@ namespace Server.Network
else
{
WriteConsole("Exceeded hue picker cap, disconnecting...");
Disconnect();
Disconnect("Exceeded hue picker cap.");
}
}
@ -330,7 +424,7 @@ namespace Server.Network
else
{
WriteConsole("Exceeded gump cap, disconnecting...");
Disconnect();
Disconnect("Exceeded gump cap.");
}
}
@ -357,16 +451,6 @@ namespace Server.Network
public override string ToString() => _toString;
public static void Pause()
{
NetworkState.Pause(ref NetworkState);
}
public static void Resume()
{
NetworkState.Resume(ref NetworkState);
}
public bool GetSendBuffer(out CircularBuffer<byte> cBuffer)
{
var result = SendPipe.Writer.TryGetMemory();
@ -377,19 +461,27 @@ namespace Server.Network
public void Send(ReadOnlySpan<byte> span)
{
if (span == null)
if (span == null || Connection == null || BlockAllPackets)
{
return;
}
var length = span.Length;
if (Connection == null || BlockAllPackets || length <= 0 || !GetSendBuffer(out var buffer))
if (length <= 0 || !GetSendBuffer(out var buffer))
{
return;
}
try
{
PacketSendProfile prof = null;
if (Core.Profiling)
{
prof = PacketSendProfile.Acquire(span[0]);
prof.Start();
}
if (_packetEncoder != null)
{
_packetEncoder(span, buffer, out length);
@ -406,14 +498,16 @@ namespace Server.Network
FlushPending.Enqueue(this);
_flushQueued = true;
}
prof?.Finish();
}
catch (Exception ex)
{
#if DEBUG
Console.WriteLine(ex);
TraceException(ex);
#endif
Disconnect();
TraceException(ex);
Disconnect("Exception while sending.");
}
}
@ -431,74 +525,359 @@ namespace Server.Network
{
byte[] buffer = p.Compile(CompressionEnabled, out var length);
if (buffer.Length > 0 && length > 0)
if (buffer == null)
{
var result = writer.TryGetMemory();
if (result.IsClosed)
WriteConsole("null buffer send, disconnecting...", this);
using (StreamWriter op = new StreamWriter("null_send.log", true))
{
p.OnSend();
return;
op.WriteLine("{0} Client: {1}: null buffer send, disconnecting...", DateTime.UtcNow, this);
op.WriteLine(new System.Diagnostics.StackTrace());
}
if (result.Length >= length)
{
result.CopyFrom(buffer.AsSpan(0, length));
writer.Advance((uint)length);
Disconnect("Attempted to send null packet buffer.");
return;
}
if (!_flushQueued)
{
FlushPending.Enqueue(this);
_flushQueued = true;
}
}
else
if (buffer.Length <= 0 || length <= 0)
{
p.OnSend();
return;
}
PacketSendProfile prof = null;
if (Core.Profiling)
{
prof = PacketSendProfile.Acquire(p.PacketID);
prof.Start();
}
var result = writer.TryGetMemory();
if (result.Length >= length)
{
result.CopyFrom(buffer.AsSpan(0, length));
writer.Advance((uint)length);
if (!_flushQueued)
{
WriteConsole("Too much data pending, disconnecting...");
Disconnect();
FlushPending.Enqueue(this);
_flushQueued = true;
}
}
else
{
WriteConsole("Didn't write anything!");
WriteConsole("Too much data pending, disconnecting...");
Disconnect("Too much data pending.");
}
prof?.Finish(length);
}
catch (Exception ex)
{
#if DEBUG
Console.WriteLine(ex);
TraceException(ex);
#endif
Disconnect();
}
finally
{
p.OnSend();
TraceException(ex);
Disconnect("Exception while sending.");
}
}
internal void Start()
{
if (Connection == null || _running)
if (Interlocked.CompareExchange(ref _running, 1, 0) == 1 || Connection == null)
{
return;
}
_running = true;
_parserState = ParserState.AwaitingNextPacket;
_protocolState = ProtocolState.AwaitingSeed;
ThreadPool.UnsafeQueueUserWorkItem(RecvTask, null);
ThreadPool.UnsafeQueueUserWorkItem(SendTask, null);
}
// Return true if there was any data to be processed. False otherwise. Used for idle detection.
public bool HandleReceive()
{
if (!Running)
{
return false;
}
bool active = false;
var reader = RecvPipe.Reader;
try
{
// Process as many packets as we can synchronously
while (Running && _parserState != ParserState.Error && _protocolState != ProtocolState.Error)
{
var result = reader.TryRead();
var length = result.Length;
if (length <= 0)
{
break;
}
// There was at least some data found, so it's not idle.
active = true;
var packetReader = new CircularBufferReader(result.Buffer);
var packetId = packetReader.ReadByte();
int packetLength = length;
// These can arrive at any time and are only informational
if (IncomingPackets.IsInfoPacket(packetId))
{
_parserState = ParserState.ProcessingPacket;
_parserState = HandlePacket(packetReader, packetId, length, out packetLength);
}
else
{
switch (_protocolState)
{
case ProtocolState.Uninitialized:
{
HandleError(packetId, packetLength);
return true;
}
case ProtocolState.AwaitingSeed:
{
if (packetId == 0xEF)
{
_parserState = ParserState.ProcessingPacket;
_parserState = HandlePacket(packetReader, packetId, length, out packetLength);
if (_parserState == ParserState.AwaitingNextPacket)
{
_protocolState = ProtocolState.LoginServer_AwaitingLogin;
}
}
else if (length >= 4)
{
int seed = (packetId << 24) | (packetReader.ReadByte() << 16) | (packetReader.ReadByte() << 8) | packetReader.ReadByte();
if (seed == 0)
{
HandleError(0, 0);
return true;
}
_seed = seed;
packetLength = 4;
_parserState = ParserState.AwaitingNextPacket;
_protocolState = ProtocolState.GameServer_AwaitingGameServerLogin;
}
else
{
_parserState = ParserState.AwaitingPartialPacket;
}
break;
}
case ProtocolState.LoginServer_AwaitingLogin:
{
if (packetId != 0xCF && packetId != 0x80)
{
WriteConsole("Possible encrypted client detected, disconnecting...");
HandleError(packetId, packetLength);
return true;
}
_parserState = ParserState.ProcessingPacket;
_parserState = HandlePacket(packetReader, packetId, length, out packetLength);
if (_parserState == ParserState.AwaitingNextPacket)
{
_protocolState = ProtocolState.LoginServer_AwaitingServerSelect;
}
break;
}
case ProtocolState.LoginServer_AwaitingServerSelect:
{
if (packetId != 0xA0)
{
HandleError(packetId, packetLength);
return true;
}
_parserState = ParserState.ProcessingPacket;
_parserState = HandlePacket(packetReader, packetId, length, out packetLength);
if (_parserState == ParserState.AwaitingNextPacket)
{
_protocolState = ProtocolState.LoginServer_ServerSelectAck;
Disconnect(string.Empty);
}
break;
}
case ProtocolState.LoginServer_ServerSelectAck:
{
// The code should never arrive here unless a packet is sent after server select.
HandleError(packetId, packetLength);
return true;
}
case ProtocolState.GameServer_AwaitingGameServerLogin:
{
if (packetId != 0x91)
{
HandleError(packetId, packetLength);
return true;
}
_parserState = ParserState.ProcessingPacket;
_parserState = HandlePacket(packetReader, packetId, length, out packetLength);
if (_parserState == ParserState.AwaitingNextPacket)
{
_protocolState = ProtocolState.GameServer_LoggedIn;
}
break;
}
case ProtocolState.GameServer_LoggedIn:
{
_parserState = ParserState.ProcessingPacket;
_parserState = HandlePacket(packetReader, packetId, length, out packetLength);
break;
}
}
}
if (_parserState == ParserState.AwaitingNextPacket)
{
reader.Advance((uint)packetLength);
}
else if (_parserState == ParserState.AwaitingPartialPacket || _parserState == ParserState.Throttled)
{
break;
}
else
{
HandleError(packetId, packetLength);
break;
}
}
reader.Commit();
}
catch (Exception ex)
{
#if DEBUG
Console.WriteLine(ex);
#endif
TraceException(ex);
Disconnect("Exception during HandleReceive");
}
return active;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void HandleError(byte packetId, int packetLength)
{
var msg =
$"{this} entered bad state on packet 0x{packetId:X2} with length {packetLength} while in protocol state {_protocolState} and parser state {_parserState}";
Disconnect(msg);
_parserState = ParserState.Error;
_protocolState = ProtocolState.Error;
}
/*
* length is the total buffer length. We might be able to use packetReader.Capacity() instead.
* packetLength is the length of the packet that this function actually found.
*/
private ParserState HandlePacket(CircularBufferReader packetReader, byte packetId, int length, out int packetLength)
{
PacketHandler handler = GetHandler(packetId);
if (handler == null)
{
WriteConsole($"received unknown packet 0x{packetId:X2} while in state {_protocolState}");
packetLength = length;
return ParserState.Error;
}
packetLength = handler.Length;
if (packetLength <= 0)
{
// Variable length packet. See if we have pulled in the length.
if (length < 3)
{
return ParserState.AwaitingPartialPacket;
}
packetLength = packetReader.ReadUInt16();
if (packetLength < 3)
{
return ParserState.Error;
}
}
// Not enough data, let's wait for more to come in
if (length < packetLength)
{
return ParserState.AwaitingPartialPacket;
}
if (handler.Ingame)
{
if (Mobile == null)
{
WriteConsole($"received packet 0x{packetId:X2} before having been attached to a mobile");
return ParserState.Error;
}
if (Mobile.Deleted)
{
return ParserState.Error;
}
}
ThrottlePacketCallback throttler = handler.ThrottleCallback;
if (throttler != null)
{
if (!throttler(packetId, this, out bool drop))
{
return drop ? ParserState.AwaitingNextPacket : ParserState.Throttled;
}
SetPacketTime(packetId);
}
PacketReceiveProfile prof = null;
if (Core.Profiling)
{
prof = PacketReceiveProfile.Acquire(packetId);
prof?.Start();
}
UpdatePacketCount(packetId);
handler.OnReceive(this, packetReader, ref packetLength);
prof?.Finish(packetLength);
return ParserState.AwaitingNextPacket;
}
private async void SendTask(object state)
{
var reader = SendPipe.Reader;
try
{
while (_running)
while (Running)
{
_sendState = SendState.AwaitingData;
var result = await reader.Read();
if (result.IsClosed)
if (result.IsClosed || !Running)
{
break;
}
@ -508,7 +887,11 @@ namespace Server.Network
continue;
}
var bytesWritten = await Connection.SendAsync(result.Buffer, SocketFlags.None).ConfigureAwait(false);
_sendState = SendState.Sending;
var bytesWritten = await Connection.SendAsync(result.Buffer, SocketFlags.None);
_sendState = SendState.SendCompleted;
if (bytesWritten > 0)
{
@ -516,17 +899,48 @@ namespace Server.Network
reader.Advance((uint)bytesWritten);
}
}
// Grab any remaining data and flush it
var data = reader.TryRead();
if (data.Length > 0)
{
_sendState = SendState.Sending;
Connection.Send(data.Buffer, SocketFlags.None);
reader.Advance((uint)data.Length);
}
}
catch (SocketException ex)
{
// If the user closes the connection (or the recv side does)
// between the check for m_Running above and the call to SendAsync,
// we can still get a socket exception here. That's ok.
#if DEBUG
Console.WriteLine(ex);
#endif
}
catch (Exception ex)
{
#if DEBUG
Console.WriteLine(ex);
TraceException(ex);
#endif
TraceException(ex);
}
finally
{
Disconnect();
try
{
Connection.Shutdown(SocketShutdown.Both);
Connection.Close();
}
catch (Exception ex)
{
TraceException(ex);
}
Disconnect("Exiting SendTask.");
_sendState = SendState.Exited;
}
}
@ -543,16 +957,12 @@ namespace Server.Network
try
{
while (_running)
while (Running)
{
if (NetworkState == NetworkState.PauseState)
{
continue;
}
_recvState = RecvState.AwaitingMemory;
var result = await writer.GetMemory();
if (result.IsClosed)
if (result.IsClosed || !Running)
{
break;
}
@ -562,12 +972,16 @@ namespace Server.Network
continue;
}
var bytesWritten = await socket.ReceiveAsync(result.Buffer, SocketFlags.None).ConfigureAwait(false);
_recvState = RecvState.AwaitingRecv;
var bytesWritten = await socket.ReceiveAsync(result.Buffer, SocketFlags.None);
if (bytesWritten <= 0)
{
break;
}
_recvState = RecvState.DataReceived;
DecodePacket(result.Buffer, ref bytesWritten);
writer.Advance((uint)bytesWritten);
@ -575,17 +989,27 @@ namespace Server.Network
// No need to flush
}
Disconnect(string.Empty);
}
catch (SocketException ex)
{
#if DEBUG
Console.WriteLine(ex);
#endif
Disconnect(string.Empty);
}
catch (Exception ex)
{
#if DEBUG
Console.WriteLine(ex);
TraceException(ex);
#endif
Disconnect("RecvTask exited unexpectedly.");
TraceException(ex);
}
finally
{
Disconnect();
_recvState = RecvState.Exited;
}
}
@ -604,56 +1028,6 @@ namespace Server.Network
return count;
}
public bool HandleReceive()
{
if (Connection == null || !_running)
{
return false;
}
try
{
var reader = RecvPipe.Reader;
// Process as many packets as we can synchronously
while (true)
{
var result = reader.TryRead();
if (result.IsClosed || result.Length <= 0)
{
return false;
}
var bytesProcessed = this.ProcessPacket(result.Buffer);
if (bytesProcessed <= 0)
{
// Error
// TODO: Throw exception instead?
if (bytesProcessed < 0)
{
Disconnect();
return false;
}
return true;
}
reader.Advance((uint)bytesProcessed);
}
}
catch (Exception ex)
{
#if DEBUG
Console.WriteLine(ex);
TraceException(ex);
#endif
Disconnect();
return false;
}
}
public void Flush()
{
if (Connection != null)
@ -664,12 +1038,20 @@ namespace Server.Network
_flushQueued = false;
}
public static int FlushAll()
public static void FlushAll()
{
while (FlushPending.Count != 0)
{
FlushPending.Dequeue()?.Flush();
}
}
public static int Slice()
{
int count = 0;
while (FlushPending.TryDequeue(out var ns))
while (FlushPending.Count != 0)
{
ns.Flush();
FlushPending.Dequeue()?.Flush();
count++;
}
@ -677,7 +1059,6 @@ namespace Server.Network
{
ns.Dispose();
TcpServer.Instances.Remove(ns);
count++;
}
return count;
@ -688,7 +1069,7 @@ namespace Server.Network
if (Connection != null && _nextActivityCheck - curTicks < 0)
{
WriteConsole("Disconnecting due to inactivity...");
Disconnect();
Disconnect("Disconnecting due to inactivity.");
}
}
@ -709,19 +1090,7 @@ namespace Server.Network
}
}
public bool CheckEncrypted(int packetID)
{
if (!SentFirstPacket && packetID != 0xF0 && packetID != 0xF1 && packetID != 0xCF && packetID != 0x80 &&
packetID != 0x91 && packetID != 0xA4 && packetID != 0xEF)
{
WriteConsole("Encrypted client detected, disconnecting");
Disconnect();
return true;
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public PacketHandler GetHandler(int packetID) => IncomingPackets.GetHandler(packetID);
public static void TraceException(Exception ex)
@ -744,41 +1113,64 @@ namespace Server.Network
Console.WriteLine(ex);
}
public void Disconnect()
public void Disconnect(string reason)
{
if (Connection == null || !_running)
if (Interlocked.CompareExchange(ref _running, 0, 1) == 0)
{
return;
}
_running = false;
try
{
Connection.Shutdown(SocketShutdown.Both);
if (_disconnectReason != string.Empty)
{
throw new Exception("Attempted to disconnect a netstate twice.");
}
}
catch (SocketException ex)
catch (Exception ex)
{
TraceException(ex);
}
try
_disconnectReason = reason;
if (Connection == null)
{
Connection.Close();
}
catch (SocketException ex)
{
TraceException(ex);
return;
}
Disposed.Enqueue(this);
}
public void TraceDisconnect()
{
if (_disconnectReason == string.Empty)
{
return;
}
try
{
using StreamWriter op = new StreamWriter("network-disconnects.log", true);
op.WriteLine($"# {DateTime.UtcNow}");
op.WriteLine($"NetState: {this}");
op.WriteLine(_disconnectReason);
op.WriteLine();
op.WriteLine();
}
catch (Exception ex)
{
TraceException(ex);
}
}
private void Dispose()
{
Connection = null;
TraceDisconnect();
RecvPipe.Writer.Flush();
RecvPipe.Writer.Close();
SendPipe.Writer.Close();
var m = Mobile;