fix(core): Tightens the network stack (#479)

- [X] Removes network pause/resume
- [X] Adds back packet profiler
- [X] Adds state machine to keep track and trace netstates
- [X] Changes NetState.Running back to using an interlock exchange
- [X] Adds preliminary packet throttling support


### Packet Throttling
- `[GetThrottle <packetId>` to get the delay in milliseconds for that packet
- `[SetThrottle <packetId> <delay>` to set the delay in milliseconds for that packet
The settings are saved to `Configuration/throttles.json`

Projects/UOContent/Accounting/AccountAttackLimiter.cs

@@ -24,23 +24,26 @@ namespace Server.Accounting
                 return;
             }
 
-            IncomingPackets.RegisterThrottler(0x80, Throttle_Callback);
-            IncomingPackets.RegisterThrottler(0x91, Throttle_Callback);
-            IncomingPackets.RegisterThrottler(0xCF, Throttle_Callback);
+            IncomingPackets.RegisterThrottler(0x80, Throttle);
+            IncomingPackets.RegisterThrottler(0x91, Throttle);
+            IncomingPackets.RegisterThrottler(0xCF, Throttle);
         }
 
-        public static TimeSpan Throttle_Callback(NetState ns)
+        public static bool Throttle(int packetId, NetState ns, out bool drop)
         {
             var accessLog = FindAccessLog(ns);
 
             if (accessLog == null)
             {
-                return TimeSpan.Zero;
+                drop = false;
+                return true;
             }
 
             var date = DateTime.UtcNow;
             var access = accessLog.LastAccessTime + ComputeThrottle(accessLog.Counts);
-            return date >= access ? TimeSpan.Zero : date - access;
+            var allow = date >= access;
+            drop = !allow;
+            return allow;
         }
 
         public static InvalidAccountAccessLog FindAccessLog(NetState ns)

Projects/UOContent/Accounting/AccountHandler.cs

@@ -216,7 +216,7 @@ namespace Server.Misc
         {
             if (!(state.Account is Account acct))
             {
-                state.Disconnect();
+                state.Disconnect("Attempted to delete a character but the account could not be found.");
                 return;
             }
 

Projects/UOContent/Commands/Generic/Commands/Commands.cs

@@ -911,14 +911,13 @@ namespace Server.Commands.Generic
                 if (list.Count > 20)
                 {
                     CommandLogging.Enabled = false;
-                    NetState.Pause();
+                    NetState.FlushAll();
                 }
 
                 base.ExecuteList(e, list);
 
                 if (list.Count > 20)
                 {
-                    NetState.Resume();
                     flushToLog = true;
                     CommandLogging.Enabled = true;
                 }
@@ -1250,7 +1249,7 @@ namespace Server.Commands.Generic
 
                         AddResponse($"They have been {(m_Ban ? "banned" : "kicked")}.");
 
-                        targState.Disconnect();
+                        targState.Disconnect($"Banned by {from}.");
 
                         if (m_Ban)
                         {

Projects/UOContent/Commands/Handlers.cs

@@ -201,15 +201,13 @@ namespace Server.Commands
                     list.Count == 1 ? "" : "s"
                 );
 
-                NetState.Pause();
+                NetState.FlushAll();
 
                 for (var i = 0; i < list.Count; ++i)
                 {
                     list[i].Delete();
                 }
 
-                NetState.Resume();
-
                 from.SendMessage("You have deleted {0} object{1}.", list.Count, list.Count == 1 ? "" : "s");
             }
             else

Projects/UOContent/Commands/Object Creation/Decorate.cs

@@ -5,6 +5,7 @@ using Server.Engines.Quests.Haven;
 using Server.Engines.Quests.Necro;
 using Server.Engines.Spawners;
 using Server.Items;
+using Server.Network;
 using Server.Utilities;
 
 namespace Server.Commands
@@ -27,6 +28,8 @@ namespace Server.Commands
 
             m_Mobile.SendMessage("Generating world decoration, please wait.");
 
+            NetState.FlushAll();
+
             Generate("Data/Decoration/Britannia", Map.Trammel, Map.Felucca);
             Generate("Data/Decoration/Trammel", Map.Trammel);
             Generate("Data/Decoration/Felucca", Map.Felucca);

Projects/UOContent/Commands/Object Creation/DecorateMag.cs

@@ -6,6 +6,7 @@ using Server.Engines.Quests.Haven;
 using Server.Engines.Quests.Necro;
 using Server.Engines.Spawners;
 using Server.Items;
+using Server.Network;
 using Server.Utilities;
 
 namespace Server.Commands
@@ -28,6 +29,8 @@ namespace Server.Commands
 
             m_Mobile.SendMessage("Generating world decoration, please wait.");
 
+            NetState.FlushAll();
+
             Generate("Data/Decoration/RuinedMaginciaTram", Map.Trammel);
             Generate("Data/Decoration/RuinedMaginciaFel", Map.Felucca);
 

Projects/UOContent/Commands/Object Creation/GenTeleporter.cs

@@ -4,6 +4,7 @@ using System.IO;
 using System.Text.Json.Serialization;
 using Server.Items;
 using Server.Json;
+using Server.Network;
 
 namespace Server.Commands
 {
@@ -68,6 +69,9 @@ namespace Server.Commands
             var from = e.Mobile;
 
             from.SendMessage("Generating teleporters, please wait.");
+
+            NetState.FlushAll();
+
             var c = new TeleportersCreator();
 
             if (!ProcessTeleporterData(from, c.CreateTeleporter))

Projects/UOContent/Commands/SignParser.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 using System.IO;
 using Server.Items;
+using Server.Network;
 
 namespace Server.Commands
 {
@@ -28,6 +29,8 @@ namespace Server.Commands
                 var list = new List<SignEntry>();
                 from.SendMessage("Generating signs, please wait.");
 
+                NetState.FlushAll();
+
                 using (var ip = new StreamReader(cfg))
                 {
                     string line;

Projects/UOContent/Engines/Doom/LeverPuzzle/LeverPuzzleController.cs

@@ -206,16 +206,14 @@ namespace Server.Engines.Doom
             eable.Free();
 
             e.Mobile.SendMessage("Generating Lamp Room puzzle...");
+
+            NetState.FlushAll();
+
             new LeverPuzzleController().MoveToWorld(lp_Center, Map.Malas);
 
-            if (!installed)
-            {
-                e.Mobile.SendMessage("There was a problem generating the puzzle.");
-            }
-            else
-            {
-                e.Mobile.SendMessage("Lamp room puzzle successfully generated.");
-            }
+            e.Mobile.SendMessage(
+                !installed ? "There was a problem generating the puzzle." : "Lamp room puzzle successfully generated."
+            );
         }
 
         public static Item AddLeverPuzzlePart(int[] loc, Item newitem)

Projects/UOContent/Gumps/AdminGump.cs

@@ -1762,7 +1762,7 @@ namespace Server.Gumps
             {
                 if (!ban)
                 {
-                    NetState.Pause();
+                    NetState.FlushAll();
                 }
 
                 for (var i = 0; i < rads.Count; ++i)
@@ -1796,11 +1796,6 @@ namespace Server.Gumps
                     }
                 }
 
-                if (!ban)
-                {
-                    NetState.Resume();
-                }
-
                 from.SendGump(
                     new NoticeGump(
                         1060637,
@@ -2365,7 +2360,7 @@ namespace Server.Gumps
 
                                             if (!hasAccess)
                                             {
-                                                ns.Disconnect();
+                                                ns.Disconnect("Server has been locked down.");
                                                 ++count;
                                             }
                                         }
@@ -3736,7 +3731,7 @@ namespace Server.Gumps
                                             "kicking",
                                             CommandLogging.Format(m)
                                         );
-                                        ns.Disconnect();
+                                        ns.Disconnect($"Kicked by {from}.");
                                         notice = "They have been kicked.";
                                     }
                                     else
@@ -3762,7 +3757,7 @@ namespace Server.Gumps
 
                                         var ns = m.NetState;
 
-                                        ns?.Disconnect();
+                                        ns?.Disconnect($"Banned by {from}.");
 
                                         notice = "They have been banned.";
                                     }

Projects/UOContent/Gumps/ClientGump.cs

@@ -272,7 +272,7 @@ namespace Server.Gumps
                         {
                             focus.Say("I've been kicked!");
 
-                            m_State.Disconnect();
+                            m_State.Disconnect($"Kicked by ${from}.");
 
                             CommandLogging.WriteLine(
                                 from,

Projects/UOContent/Items/Skill Items/Camping/Bedroll.cs

@@ -121,7 +121,7 @@ namespace Server.Items
                     pm.PlaceInBackpack(m_Bedroll);
 
                     pm.BedrollLogout = true;
-                    sender.Disconnect();
+                    sender.Disconnect("Used a bedroll to log out.");
                 }
 
                 Campfire.RemoveEntry(m_Entry);

Projects/UOContent/Misc/ClientVerification.cs

@@ -172,7 +172,7 @@ namespace Server.Misc
             if (ns.Connection != null)
             {
                 ns.WriteConsole("Disconnecting, bad version");
-                ns.Disconnect();
+                ns.Disconnect($"Invalid client version {ns.Version}.");
             }
         }
 

Projects/UOContent/Misc/DoorGenerator.cs

@@ -347,7 +347,7 @@ namespace Server
         {
             World.Broadcast(0x35, true, "Generating doors, please wait.");
 
-            NetState.Pause();
+            NetState.FlushAll();
 
             m_Map = Map.Trammel;
             m_Count = 0;
@@ -389,8 +389,6 @@ namespace Server
 
             var malasCount = m_Count;
 
-            NetState.Resume();
-
             World.Broadcast(
                 0x35,
                 true,

Projects/UOContent/Misc/HardwareInfo.cs

@@ -145,35 +145,35 @@ namespace Server
         {
             reader.ReadByte(); // 1: <4.0.1a, 2>=4.0.1a
 
-            var info = new HardwareInfo();
-
-            info.InstanceID = reader.ReadInt32();
-            info.OSMajor = reader.ReadInt32();
-            info.OSMinor = reader.ReadInt32();
-            info.OSRevision = reader.ReadInt32();
-            info.CpuManufacturer = reader.ReadByte();
-            info.CpuFamily = reader.ReadInt32();
-            info.CpuModel = reader.ReadInt32();
-            info.CpuClockSpeed = reader.ReadInt32();
-            info.CpuQuantity = reader.ReadByte();
-            info.PhysicalMemory = reader.ReadInt32();
-            info.ScreenWidth = reader.ReadInt32();
-            info.ScreenHeight = reader.ReadInt32();
-            info.ScreenDepth = reader.ReadInt32();
-            info.DXMajor = reader.ReadInt16();
-            info.DXMinor = reader.ReadInt16();
-            info.VCDescription = reader.ReadLittleUniSafe(64);
-            info.VCVendorID = reader.ReadInt32();
-            info.VCDeviceID = reader.ReadInt32();
-            info.VCMemory = reader.ReadInt32();
-            info.Distribution = reader.ReadByte();
-            info.ClientsRunning = reader.ReadByte();
-            info.ClientsInstalled = reader.ReadByte();
-            info.PartialInstalled = reader.ReadByte();
-            info.Language = reader.ReadLittleUniSafe(4);
-            info.Unknown = reader.ReadAsciiSafe(64);
-
-            info.TimeReceived = DateTime.UtcNow;
+            var info = new HardwareInfo
+            {
+                InstanceID = reader.ReadInt32(),
+                OSMajor = reader.ReadInt32(),
+                OSMinor = reader.ReadInt32(),
+                OSRevision = reader.ReadInt32(),
+                CpuManufacturer = reader.ReadByte(),
+                CpuFamily = reader.ReadInt32(),
+                CpuModel = reader.ReadInt32(),
+                CpuClockSpeed = reader.ReadInt32(),
+                CpuQuantity = reader.ReadByte(),
+                PhysicalMemory = reader.ReadInt32(),
+                ScreenWidth = reader.ReadInt32(),
+                ScreenHeight = reader.ReadInt32(),
+                ScreenDepth = reader.ReadInt32(),
+                DXMajor = reader.ReadInt16(),
+                DXMinor = reader.ReadInt16(),
+                VCDescription = reader.ReadLittleUniSafe(64),
+                VCVendorID = reader.ReadInt32(),
+                VCDeviceID = reader.ReadInt32(),
+                VCMemory = reader.ReadInt32(),
+                Distribution = reader.ReadByte(),
+                ClientsRunning = reader.ReadByte(),
+                ClientsInstalled = reader.ReadByte(),
+                PartialInstalled = reader.ReadByte(),
+                Language = reader.ReadLittleUniSafe(4),
+                Unknown = reader.ReadAsciiSafe(64),
+                TimeReceived = DateTime.UtcNow
+            };
 
             if (state.Account is Account acct)
             {

Projects/UOContent/Misc/PacketThrottles.cs

@@ -0,0 +1,157 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using Server.Json;
+using Server.Mobiles;
+
+namespace Server.Network
+{
+    public static class PacketThrottles
+    {
+        // Delay in milliseconds
+        private static readonly int[] Delays = new int[0x100];
+        private static string ThrottlesConfiguration = "Configuration/throttles.json";
+
+        public static void Initialize()
+        {
+            CommandSystem.Register("GetThrottle", AccessLevel.Administrator, GetThrottle);
+            CommandSystem.Register("SetThrottle", AccessLevel.Administrator, SetThrottle);
+
+            var configPath = ThrottlesConfiguration;
+            var path = Path.Join(Core.BaseDirectory, configPath);
+
+            if (File.Exists(path))
+            {
+                var throttles = JsonConfig.Deserialize<SortedDictionary<string, int>>(path);
+                foreach (var (k, v) in throttles)
+                {
+                    if (!Utility.ToInt32(k, out var packetId))
+                    {
+                        Utility.PushColor(ConsoleColor.DarkYellow);
+                        Console.WriteLine("Packet Throttles: Error deserializing {0} from {1}", k, configPath);
+                        Utility.PopColor();
+                        continue;
+                    }
+
+                    Delays[packetId] = v;
+                }
+            }
+            else
+            {
+                Delays[0x03] = 5; // Speech
+                Delays[0xAD] = 5; // Speech
+                Delays[0x75] = 500; // Rename request
+            }
+
+            for (int i = 0; i < 0x100; i++)
+            {
+                if (Delays[i] > 0)
+                {
+                    IncomingPackets.RegisterThrottler(i, Throttle);
+                }
+            }
+
+            SaveDelays();
+        }
+
+        [Usage("GetThrottle <packetID>")]
+        [Description("Gets throttle for the given packet.")]
+        public static void GetThrottle(CommandEventArgs e)
+        {
+            if (e.Length != 1)
+            {
+                e.Mobile.SendMessage("Invalid Command Format. Should be [GetThrottle <packetID>");
+                return;
+            }
+
+            int packetID = e.GetInt32(0);
+
+            if (packetID < 0 || packetID > 0x100)
+            {
+                e.Mobile.SendMessage("Invalid Command Format. PacketID must be between 0 and 0x100.");
+                return;
+            }
+
+
+            e.Mobile.SendMessage("Packet 0x{0:X} throttle is currently {1}ms.", packetID, Delays[packetID]);
+        }
+
+        [Usage("SetThrottle <packetID> <timeInMilliseconds>")]
+        [Description("Sets a throttle for the given packet.")]
+        public static void SetThrottle(CommandEventArgs e)
+        {
+            if (e.Length != 2)
+            {
+                e.Mobile.SendMessage("Invalid Command Format. Should be [SetThrottle <packetID> <timeInMilliseconds>");
+                return;
+            }
+
+            int packetID = e.GetInt32(0);
+            int delay = e.GetInt32(1);
+
+            if (packetID < 0 || packetID > 0x100)
+            {
+                e.Mobile.SendMessage("Invalid Command Format. PacketID must be between 0 and 0x100.");
+                return;
+            }
+
+
+            if (delay > 5000)
+            {
+                e.Mobile.SendMessage("Invalid Command Format. Delay cannot exceed 5000 milliseconds.");
+                return;
+            }
+
+            long oldDelay = Delays[packetID];
+
+            if (oldDelay == 0 && delay > 0)
+            {
+                IncomingPackets.RegisterThrottler(packetID, Throttle);
+            }
+            else if (oldDelay > 0 && delay == 0)
+            {
+                IncomingPackets.RegisterThrottler(packetID, null);
+            }
+
+            Delays[packetID] = delay;
+            SaveDelays();
+        }
+
+        private static void SaveDelays()
+        {
+            SortedDictionary<string, int> table = new();
+            for (var i = 0; i < Delays.Length; i++)
+            {
+                var delay = Delays[i];
+
+                if (delay != 0)
+                {
+                    table[$"0x{i:X2}"] = delay;
+                }
+            }
+
+            var configPath = ThrottlesConfiguration;
+            var path = Path.Join(Core.BaseDirectory, configPath);
+            JsonConfig.Serialize(path, table);
+        }
+
+        public static bool Throttle(int packetID, NetState ns, out bool drop)
+        {
+            if (ns.Mobile is not PlayerMobile player || player.AccessLevel >= AccessLevel.Counselor)
+            {
+                drop = false;
+                return true;
+            }
+
+            if (Core.TickCount < ns.GetPacketDelay(packetID) + Delays[packetID])
+            {
+                drop = true;
+                return false;
+            }
+
+            drop = false;
+            return true;
+        }
+    }
+}

Projects/UOContent/Misc/ProfanityProtection.cs

@@ -99,7 +99,7 @@ namespace Server.Misc
                     return true;
                 case ProfanityAction.Disconnect:
                     {
-                        from.NetState?.Disconnect();
+                        from.NetState?.Disconnect("Using profanity.");
 
                         return false;
                     }

Projects/UOContent/Mobiles/PlayerMobile.cs

@@ -1287,7 +1287,7 @@ namespace Server.Mobiles
 
                     if (from.NetState != null)
                     {
-                        Timer.DelayCall(TimeSpan.FromSeconds(1.0), from.NetState.Disconnect);
+                        Timer.DelayCall(TimeSpan.FromSeconds(1.0), from.NetState.Disconnect, "Server is locked down");
                     }
                 }
                 else if (from.AccessLevel >= AccessLevel.Administrator)

Projects/UOContent/Network/ProtocolExtensions.cs

@@ -39,11 +39,11 @@ namespace Server.Network
             if (ph.Ingame && state.Mobile == null)
             {
                 state.WriteConsole("Sent in-game packet (0xBFx{0:X2}) before having been attached to a mobile", packetID);
-                state.Disconnect();
+                state.Disconnect("Sent in-game packet before being attached to a mobile.");
             }
             else if (ph.Ingame && state.Mobile.Deleted)
             {
-                state.Disconnect();
+                state.Disconnect(string.Empty);
             }
             else
             {