feat: Networking v3 using epoll/kqueue (#813)

* Replaces networking internals with epoll/kqueue

Projects/Server/Main.cs

@@ -55,7 +55,8 @@ namespace Server
         public static readonly bool IsDarwin = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
         public static readonly bool IsFreeBSD = RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD);
         public static readonly bool IsLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || IsFreeBSD;
-        public static readonly bool Unix = IsDarwin || IsFreeBSD || IsLinux;
+        public static readonly bool IsBSD = IsDarwin || IsFreeBSD;
+        public static readonly bool Unix = IsBSD || IsLinux;
 
         private const string AssembliesConfiguration = "Data/assemblies.json";
 
@@ -516,7 +517,6 @@ namespace Server
 
                     // Handle networking
                     TcpServer.Slice();
-                    NetState.HandleAllReceives();
                     NetState.Slice();
 
                     // Execute captured post-await methods (like Timer.Pause)

Projects/Server/Network/ISocket.cs

@@ -23,6 +23,8 @@ namespace Server.Network
 {
     public interface ISocket
     {
+        public IntPtr Handle { get; }
+
         public EndPoint LocalEndPoint { get; }
 
         public EndPoint RemoteEndPoint { get; }
@@ -33,6 +35,8 @@ namespace Server.Network
 
         public Task<int> ReceiveAsync(IList<ArraySegment<byte>> buffer, SocketFlags flags);
 
+        public int Receive(IList<ArraySegment<byte>> buffers, SocketFlags flags);
+
         public void Shutdown(SocketShutdown how);
 
         public void Close();

Projects/Server/Network/NetState/DumpNetStates.cs

@@ -32,7 +32,7 @@ namespace Server.Network
 
             foreach (var ns in TcpServer.Instances)
             {
-                file.WriteLine($"{ns}, {ns._recvState}, {ns._sendState}, {ns._protocolState}, {ns._parserState}");
+                file.WriteLine($"{ns}, {ns._protocolState}, {ns._parserState}");
             }
         }
     }

Projects/Server/Network/NetState/NetState.cs

@@ -21,7 +21,7 @@ using System.IO;
 using System.Net;
 using System.Net.Sockets;
 using System.Runtime.CompilerServices;
-using System.Threading;
+using System.Runtime.InteropServices;
 using Server.Accounting;
 using Server.Diagnostics;
 using Server.Gumps;
@@ -43,12 +43,15 @@ namespace Server.Network
 
         private const int RecvPipeSize = 1024 * 64;
         private const int SendPipeSize = 1024 * 256;
-        private static int GumpCap = 512;
+        private const int GumpCap = 512;
-        private static int HuePickerCap = 512;
+        private const int HuePickerCap = 512;
-        private static int MenuCap = 512;
+        private const int MenuCap = 512;
-        private static int PacketPerSecondThreshold = 3000;
+        private const int PacketPerSecondThreshold = 3000;
 
+        private static NetState[] _polledStates = new NetState[2048];
+        private static readonly IPollGroup _pollGroup = PollGroup.Create();
         private static readonly Queue<NetState> FlushPending = new(2048);
+        private static readonly Queue<NetState> FlushedPartials = new(2048);
         private static readonly ConcurrentQueue<NetState> Disposed = new();
 
         public static NetStateCreatedCallback CreatedCallback { get; set; }
@@ -56,7 +59,7 @@ namespace Server.Network
         private readonly string _toString;
         private ClientVersion _version;
         private long _nextActivityCheck;
-        private int _running;
+        private bool _running = true;
         private volatile DecodePacket _packetDecoder;
         private volatile EncodePacket _packetEncoder;
         private bool _flushQueued;
@@ -66,14 +69,12 @@ namespace Server.Network
 
         internal int _authId;
         internal int _seed;
-        internal ParserState _parserState = ParserState.Uninitialized;
+        internal ParserState _parserState = ParserState.AwaitingNextPacket;
-        internal ProtocolState _protocolState = ProtocolState.Uninitialized;
+        internal ProtocolState _protocolState = ProtocolState.AwaitingSeed;
-        internal RecvState _recvState = RecvState.Uninitialized;
+        internal GCHandle _handle;
-        internal SendState _sendState = SendState.Uninitialized;
 
         internal enum ParserState
         {
-            Uninitialized,
             AwaitingNextPacket,
             AwaitingPartialPacket,
             ProcessingPacket,
@@ -83,7 +84,6 @@ namespace Server.Network
 
         internal enum ProtocolState
         {
-            Uninitialized,
             AwaitingSeed, // Based on the way the seed arrives, we know if this is a login server or a game server connection
 
             LoginServer_AwaitingLogin,
@@ -96,32 +96,6 @@ namespace Server.Network
             Error
         }
 
-        internal enum RecvState
-        {
-            Uninitialized,
-            AwaitingMemory,
-            AwaitingRecv,
-            DataReceived,
-            Exited,
-        }
-
-        internal enum SendState
-        {
-            Uninitialized,
-            AwaitingData,
-            Sending,
-            SendCompleted,
-            Exited,
-        }
-
-        public static void Configure()
-        {
-            GumpCap = ServerConfiguration.GetOrUpdateSetting("netstate.gumpCap", GumpCap);
-            HuePickerCap = ServerConfiguration.GetOrUpdateSetting("netstate.huePickerCap", HuePickerCap);
-            MenuCap = ServerConfiguration.GetOrUpdateSetting("netstate.menuCap", MenuCap);
-            PacketPerSecondThreshold = ServerConfiguration.GetOrUpdateSetting("netstate.packetsPerSecondThreshold", PacketPerSecondThreshold);
-        }
-
         public static void Initialize()
         {
             Timer.DelayCall(TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1.5), CheckAllAlive);
@@ -138,6 +112,7 @@ namespace Server.Network
             RecvPipe = new Pipe<byte>(GC.AllocateUninitializedArray<byte>(RecvPipeSize));
             SendPipe = new Pipe<byte>(GC.AllocateUninitializedArray<byte>(SendPipeSize));
             _nextActivityCheck = Core.TickCount + 30000;
+            ConnectedOn = Core.Now;
 
             try
             {
@@ -151,7 +126,17 @@ namespace Server.Network
                 _toString = "(error)";
             }
 
-            ConnectedOn = Core.Now;
+            _handle = GCHandle.Alloc(this);
+
+            try
+            {
+                _pollGroup.Add(this);
+            }
+            catch (Exception ex)
+            {
+                TraceException(ex);
+                Disconnect("Unable to add socket to poll group");
+            }
 
             CreatedCallback?.Invoke(this);
         }
@@ -180,12 +165,6 @@ namespace Server.Network
 
         public List<SecureTrade> Trades { get; }
 
-        public bool Running
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _running == 1;
-        }
-
         public bool Seeded { get; set; }
 
         public Pipe<byte> RecvPipe { get; }
@@ -524,33 +503,21 @@ namespace Server.Network
             }
         }
 
-        internal void Start()
-        {
-            if (Interlocked.CompareExchange(ref _running, 1, 0) == 1 || Connection == null)
-            {
-                return;
-            }
-
-            _parserState = ParserState.AwaitingNextPacket;
-            _protocolState = ProtocolState.AwaitingSeed;
-
-            ThreadPool.UnsafeQueueUserWorkItem(RecvTask, null);
-            ThreadPool.UnsafeQueueUserWorkItem(SendTask, null);
-        }
-
         public void HandleReceive()
         {
-            if (!Running)
+            if (!_running)
             {
                 return;
             }
 
+            ReceiveData();
+
             var reader = RecvPipe.Reader;
 
             try
             {
                 // Process as many packets as we can synchronously
-                while (Running && _parserState != ParserState.Error && _protocolState != ProtocolState.Error)
+                while (_running && _parserState != ParserState.Error && _protocolState != ProtocolState.Error)
                 {
                     var result = reader.TryRead();
                     var length = result.Length;
@@ -574,12 +541,6 @@ namespace Server.Network
                     {
                         switch (_protocolState)
                         {
-                            case ProtocolState.Uninitialized:
-                                {
-                                    HandleError(packetId, packetLength);
-                                    return;
-                                }
-
                             case ProtocolState.AwaitingSeed:
                                 {
                                     if (packetId == 0xEF)
@@ -803,83 +764,56 @@ namespace Server.Network
             return ParserState.AwaitingNextPacket;
         }
 
-        private async void SendTask(object state)
+        private bool Flush()
         {
+            _flushQueued = false;
+
+            if (Connection == null)
+            {
+                return true;
+            }
+
+            SendPipe.Writer.Flush();
+
             var reader = SendPipe.Reader;
+            var result = reader.TryRead();
+
+            if (result.IsClosed || result.Length == 0)
+            {
+                return true;
+            }
+
+            var bytesWritten = 0;
 
             try
             {
-                while (Running)
+                bytesWritten = Connection.Send(result.Buffer, SocketFlags.None);
-                {
-                    _sendState = SendState.AwaitingData;
-
-                    var result = await reader.Read();
-
-                    if (result.IsClosed || !Running)
-                    {
-                        break;
-                    }
-
-                    if (result.Length <= 0)
-                    {
-                        continue;
-                    }
-
-                    _sendState = SendState.Sending;
-
-                    var bytesWritten = await Connection.SendAsync(result.Buffer, SocketFlags.None);
-
-                    _sendState = SendState.SendCompleted;
-
-                    if (bytesWritten > 0)
-                    {
-                        _nextActivityCheck = Core.TickCount + 90000;
-                        reader.Advance((uint)bytesWritten);
-                    }
-                }
-
-                // Grab any remaining data and flush it
-                var data = reader.TryRead();
-
-                if (data.Length > 0)
-                {
-                    _sendState = SendState.Sending;
-                    Connection.Send(data.Buffer, SocketFlags.None);
-
-                    reader.Advance((uint)data.Length);
-                }
             }
             catch (SocketException ex)
             {
-                // If the user closes the connection (or the recv side does)
+                // Socket exceptions are generally ok, just spammy
-                // between the check for m_Running above and the call to SendAsync,
-                // we can still get a socket exception here. That's ok.
 #if DEBUG
                 Console.WriteLine(ex);
 #endif
+
+                Disconnect(string.Empty);
             }
             catch (Exception ex)
             {
 #if DEBUG
                 Console.WriteLine(ex);
 #endif
+                Disconnect($"Disconnected with error: {ex}");
                 TraceException(ex);
             }
-            finally
-            {
-                try
-                {
-                    Connection.Shutdown(SocketShutdown.Both);
-                    Connection.Close();
-                }
-                catch (Exception ex)
-                {
-                    TraceException(ex);
-                }
 
-                Disconnect("Exiting SendTask.");
+            if (bytesWritten > 0)
-                _sendState = SendState.Exited;
+            {
+                _nextActivityCheck = Core.TickCount + 90000;
+                reader.Advance((uint)bytesWritten);
             }
+
+            return bytesWritten == result.Length;
         }
 
         private void DecodePacket(ArraySegment<byte>[] buffer, ref int length)
@@ -888,47 +822,21 @@ namespace Server.Network
             _packetDecoder?.Invoke(cBuffer, ref length);
         }
 
-        private async void RecvTask(object state)
+        private void ReceiveData()
         {
-            var socket = Connection;
             var writer = RecvPipe.Writer;
+            var result = writer.TryGetMemory();
+
+            if (result.IsClosed || result.Length == 0)
+            {
+                return;
+            }
+
+            var bytesWritten = 0;
 
             try
             {
-                while (Running)
+                bytesWritten = Connection.Receive(result.Buffer, SocketFlags.None);
-                {
-                    _recvState = RecvState.AwaitingMemory;
-                    var result = await writer.GetMemory();
-
-                    if (result.IsClosed || !Running)
-                    {
-                        break;
-                    }
-
-                    if (result.Length <= 0)
-                    {
-                        continue;
-                    }
-
-                    _recvState = RecvState.AwaitingRecv;
-
-                    var bytesWritten = await socket.ReceiveAsync(result.Buffer, SocketFlags.None);
-                    if (bytesWritten <= 0)
-                    {
-                        break;
-                    }
-
-                    _recvState = RecvState.DataReceived;
-
-                    DecodePacket(result.Buffer, ref bytesWritten);
-
-                    writer.Advance((uint)bytesWritten);
-                    _nextActivityCheck = Core.TickCount + 90000;
-
-                    // No need to flush
-                }
-
-                Disconnect(string.Empty);
             }
             catch (SocketException ex)
             {
@@ -945,31 +853,20 @@ namespace Server.Network
 #if DEBUG
                 Console.WriteLine(ex);
 #endif
-                Disconnect("RecvTask exited unexpectedly.");
+                Disconnect($"Disconnected with error: {ex}");
                 TraceException(ex);
             }
-            finally
-            {
-                _recvState = RecvState.Exited;
-            }
-        }
 
-        public static void HandleAllReceives()
+            if (bytesWritten <= 0)
-        {
-            foreach (var ns in TcpServer.Instances)
             {
-                ns.HandleReceive();
+                Disconnect(string.Empty);
-            }
+                return;
-        }
-
-        public void Flush()
-        {
-            if (Connection != null)
-            {
-                SendPipe.Writer.Flush();
             }
 
-            _flushQueued = false;
+            DecodePacket(result.Buffer, ref bytesWritten);
+
+            writer.Advance((uint)bytesWritten);
+            _nextActivityCheck = Core.TickCount + 90000;
         }
 
         public static void FlushAll()
@@ -982,16 +879,36 @@ namespace Server.Network
 
         public static void Slice()
         {
-            while (FlushPending.Count != 0)
+            int count = _pollGroup.Poll(ref _polledStates);
+
+            if (count > 0)
             {
-                FlushPending.Dequeue()?.Flush();
+                for (int i = 0; i < count; i++)
+                {
+                    _polledStates[i].HandleReceive();
+                    _polledStates[i] = null;
+                }
             }
 
+            while (FlushPending.TryDequeue(out var ns))
+            {
+                if (!ns.Flush())
+                {
+                    // Incomplete data, so we need to requeue
+                    FlushedPartials.Enqueue(ns);
+                }
+            }
+
+            var hasDisposes = !Disposed.IsEmpty;
             while (Disposed.TryDequeue(out var ns))
             {
-                TcpServer.Instances.Remove(ns);
                 ns.Dispose();
             }
+
+            if (hasDisposes)
+            {
+                _pollGroup.Poll(ref _polledStates);
+            }
         }
 
         public void CheckAlive(long curTicks)
@@ -1045,11 +962,13 @@ namespace Server.Network
 
         public void Disconnect(string reason)
         {
-            if (Interlocked.CompareExchange(ref _running, 0, 1) == 0)
+            if (!_running)
             {
                 return;
             }
 
+            _running = false;
+
             try
             {
                 if (_disconnectReason != string.Empty)
@@ -1063,7 +982,6 @@ namespace Server.Network
             }
 
             _disconnectReason = reason;
-
             Disposed.Enqueue(this);
         }
 
@@ -1095,6 +1013,11 @@ namespace Server.Network
         {
             TraceDisconnect(_disconnectReason, _toString);
 
+            if (_running)
+            {
+                throw new Exception("Disconnected a NetState that is still running.");
+            }
+
 #if THREADGUARD
             if (Thread.CurrentThread != Core.Thread)
             {
@@ -1106,8 +1029,18 @@ namespace Server.Network
             }
 #endif
 
-            RecvPipe.Writer.Close();
+            TcpServer.Instances.Remove(this);
-            SendPipe.Writer.Close();
+            try
+            {
+                _pollGroup.Remove(this);
+            }
+            catch (Exception ex)
+            {
+                TraceException(ex);
+            }
+
+            Connection.Close();
+            _handle.Free();
 
             var m = Mobile;
             Mobile = null;

Projects/Server/Network/NetworkSocket.cs

@@ -32,6 +32,12 @@ namespace Server.Network
             get => _connection;
         }
 
+        public IntPtr Handle
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => _connection.Handle;
+        }
+
         public EndPoint LocalEndPoint
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -58,6 +64,10 @@ namespace Server.Network
         public Task<int> ReceiveAsync(IList<ArraySegment<byte>> buffers, SocketFlags flags) =>
             _connection.ReceiveAsync(buffers, flags);
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Receive(IList<ArraySegment<byte>> buffers, SocketFlags flags) =>
+            _connection.Receive(buffers, flags);
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void Shutdown(SocketShutdown how) => _connection.Shutdown(how);
 

Projects/Server/Network/PollGroup/EPollGroup.cs

@@ -0,0 +1,206 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2021 - ModernUO Development Team                       *
+ * Email: hi@modernuo.com                                                *
+ * File: EPollGroup.cs                                                   *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Server.Network
+{
+    public sealed class EPollGroup : IPollGroup
+    {
+        [Flags]
+        private enum epoll_flags
+        {
+            NONE = 0,
+            CLOEXEC = 0x02000000,
+            NONBLOCK = 0x04000,
+        }
+
+        [Flags]
+        private enum epoll_events : uint
+        {
+            EPOLLIN = 0x001,
+            EPOLLPRI = 0x002,
+            EPOLLOUT = 0x004,
+            EPOLLRDNORM = 0x040,
+            EPOLLRDBAND = 0x080,
+            EPOLLWRNORM = 0x100,
+            EPOLLWRBAND = 0x200,
+            EPOLLMSG = 0x400,
+            EPOLLERR = 0x008,
+            EPOLLHUP = 0x010,
+            EPOLLRDHUP = 0x2000,
+            EPOLLONESHOT = 1 << 30,
+            EPOLLET = unchecked((uint)(1 << 31))
+        }
+
+        private enum epoll_op
+        {
+            EPOLL_CTL_ADD = 1,
+            EPOLL_CTL_DEL = 2,
+            EPOLL_CTL_MOD = 3,
+        }
+
+        [StructLayout(LayoutKind.Explicit)]
+        private struct epoll_data
+        {
+            [FieldOffset(0)]
+            public int fd;
+
+            [FieldOffset(0)]
+            public IntPtr ptr;
+
+            [FieldOffset(0)]
+            public uint u32;
+
+            [FieldOffset(0)]
+            public ulong u64;
+        }
+
+        [StructLayout(LayoutKind.Sequential)]
+        private struct epoll_event
+        {
+            public epoll_events events;
+            public epoll_data data;
+        }
+
+        private static class Windows
+        {
+            [DllImport("wepoll.dll", SetLastError = true)]
+            public static extern int epoll_create1(epoll_flags flags);
+
+            [DllImport("wepoll.dll", SetLastError = true)]
+            public static extern int epoll_close(int epfd);
+
+            [DllImport("wepoll.dll", SetLastError = true)]
+            public static extern int epoll_ctl(int epfd, epoll_op op, int fd, ref epoll_event ee);
+
+            [DllImport("wepoll.dll", SetLastError = true)]
+            public static extern int epoll_wait(int epfd, [In, Out] epoll_event[] ee, int maxevents, int timeout);
+        }
+
+        private static class Linux
+        {
+            [DllImport("libc", SetLastError = true)]
+            public static extern int epoll_create1(epoll_flags flags);
+
+            [DllImport("libc", SetLastError = true)]
+            public static extern int epoll_close(int epfd);
+
+            [DllImport("libc", SetLastError = true)]
+            public static extern int epoll_ctl(int epfd, epoll_op op, int fd, ref epoll_event ee);
+
+            [DllImport("libc", SetLastError = true)]
+            public static extern int epoll_wait(int epfd, [In, Out] epoll_event[] ee, int maxevents, int timeout);
+        }
+
+        private readonly int _epHndle;
+
+        public EPollGroup()
+        {
+            _epHndle = Core.IsWindows ? Windows.epoll_create1(epoll_flags.NONE) : Linux.epoll_create1(epoll_flags.NONE);
+
+            if (_epHndle == 0)
+            {
+                throw new Exception("Unable to initialize poll group");
+            }
+        }
+
+        public void Dispose()
+        {
+            if (Core.IsWindows)
+            {
+                Windows.epoll_close(_epHndle);
+            }
+            else
+            {
+                Linux.epoll_close(_epHndle);
+            }
+        }
+
+        public void Add(NetState state)
+        {
+            var ev = new epoll_event
+            {
+                events = epoll_events.EPOLLIN | epoll_events.EPOLLERR
+            };
+
+            ev.data.ptr = (IntPtr)state._handle;
+
+            var rc = Core.IsWindows ?
+                Windows.epoll_ctl(_epHndle, epoll_op.EPOLL_CTL_ADD, (int)state.Connection.Handle, ref ev) :
+                Linux.epoll_ctl(_epHndle, epoll_op.EPOLL_CTL_ADD, (int)state.Connection.Handle, ref ev);
+
+
+            if (rc != 0)
+            {
+                throw new Exception($"epoll_ctl failed with error code {Marshal.GetLastWin32Error()}");
+            }
+        }
+
+        public void Remove(NetState state)
+        {
+            var ev = new epoll_event
+            {
+                events = epoll_events.EPOLLIN | epoll_events.EPOLLERR,
+            };
+            ev.data.ptr = (IntPtr)state._handle;
+
+            var rc = Core.IsWindows ?
+                Windows.epoll_ctl(_epHndle, epoll_op.EPOLL_CTL_DEL, (int)state.Connection.Handle, ref ev) :
+                Linux.epoll_ctl(_epHndle, epoll_op.EPOLL_CTL_DEL, (int)state.Connection.Handle, ref ev);
+
+            if (rc != 0)
+            {
+                throw new Exception($"epoll_ctl failed with error code {Marshal.GetLastWin32Error()}");
+            }
+        }
+
+        private epoll_event[] _events = new epoll_event[2048];
+
+        public int Poll(ref NetState[] states)
+        {
+            if (states.Length > _events.Length)
+            {
+                var newLength = Math.Max(states.Length, _events.Length + (_events.Length >> 2));
+                _events = new epoll_event[newLength];
+            }
+
+            var rc = Core.IsWindows ?
+                Windows.epoll_wait(_epHndle, _events, states.Length, 0) :
+                Linux.epoll_wait(_epHndle, _events, states.Length, 0);
+
+            if (rc <= 0)
+            {
+                return rc;
+            }
+
+            int count = 0;
+
+            for (int i = 0; i < rc; i++)
+            {
+                if (((GCHandle)_events[i].data.ptr).Target is not NetState state)
+                {
+                    continue;
+                }
+
+                states[count++] = state;
+            }
+
+            return count;
+        }
+
+    }
+}

Projects/Server/Network/PollGroup/KQueuePollGroup.cs

@@ -0,0 +1,248 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2021 - ModernUO Development Team                       *
+ * Email: hi@modernuo.com                                                *
+ * File: KQueuePollGroup.cs                                              *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace Server.Network
+{
+    public class KQueuePollGroup : IPollGroup
+    {
+        [StructLayout(LayoutKind.Sequential)]
+        private struct kevent
+        {
+            public IntPtr ident;
+            public kqueue_filter filter;
+            public kqueue_flags flags;
+            public kqueue_fflags fflags;
+            public IntPtr data;
+            public IntPtr udata;
+        }
+
+        [Flags]
+        private enum kqueue_filter : short
+        {
+            READ = -1,
+            WRITE = -2,
+            AIO = -3,
+            VNODE = -4,
+            PROC = -5,
+            SIGNAL = -6,
+            TIMER = -7,
+            MACHPORT = -8,
+            FS = -9,
+            USER = -10,
+            UNUSED = -11,
+            VM = -12,
+            EXCEPT = -15
+        }
+
+        [Flags]
+        private enum kqueue_flags : ushort
+        {
+            ADD = 0x0001,
+            DELETE = 0x0002,
+            ENABLE = 0x0004,
+            DISABLE = 0x0008,
+            ONESHOT = 0x0010,
+            CLEAR = 0x0020,
+            RECEIPT = 0x0040,
+            DISPATCH = 0x0080,
+            UDATA_SPECIFIC = 0x0100,
+            DISPATCH2 = (DISPATCH | UDATA_SPECIFIC),
+            VANISHED = 0x0200,
+            SYSFLAGS = 0xF000,
+            FLAG0 = 0x1000,
+            FLAG1 = 0x2000,
+            EOF = 0x8000,
+            ERROR = 0x4000
+        }
+
+        [Flags]
+        private enum kqueue_fflags : uint
+        {
+            TRIGGER = 0x01000000,
+            FFNOP = 0x00000000,
+            FFAND = 0x40000000,
+            FFOR = 0x80000000,
+            FFCOPY = 0xc0000000,
+            FFCTRLMASK = 0xc0000000,
+            FFFLAGSMASK = 0x00ffffff,
+            LOWAT = 0x00000001,
+            DELETE = 0x00000001,
+            WRITE = 0x00000002,
+            EXTEND = 0x00000004,
+            ATTRIB = 0x00000008,
+            LINK = 0x00000010,
+            RENAME = 0x00000020,
+            REVOKE = 0x00000040,
+            NONE = 0x00000080,
+        }
+
+        [StructLayout(LayoutKind.Sequential)]
+        struct timespec
+        {
+            public long tv_sec;
+            public long tv_nsec;
+
+            public timespec(long sec, long nsec)
+            {
+                tv_sec = sec;
+                tv_nsec = nsec;
+            }
+        }
+
+        private static readonly kevent[] _singleEvent = new kevent[1];
+        private static readonly IntPtr _zeroTimeoutPtr;
+        // ReSharper disable once PrivateFieldCanBeConvertedToLocalVariable
+        private static readonly timespec _zeroTimeout;
+
+        static KQueuePollGroup()
+        {
+            _zeroTimeout = new timespec(0, 0);
+            _zeroTimeoutPtr = Marshal.AllocHGlobal(Marshal.SizeOf<timespec>());
+            Marshal.StructureToPtr(_zeroTimeout, _zeroTimeoutPtr, false);
+        }
+
+        private static class BSD
+        {
+            [DllImport ("libc", SetLastError = true)]
+            public static extern int close (int fd);
+
+            [DllImport("libc", SetLastError = true)]
+            public static extern int kqueue();
+
+            [DllImport("libc", SetLastError = true)]
+            public static extern int kevent(int kq, kevent[] changelist, int nchanges, [In, Out] kevent[] eventlist, int nevents, IntPtr timeout);
+
+            public static int kevent(
+                int kq,
+                IntPtr ident,
+                kqueue_filter filter,
+                kqueue_flags flags,
+                kqueue_fflags fflags = 0,
+                IntPtr data = default,
+                IntPtr udata = default
+            )
+            {
+                _singleEvent[0] = new kevent
+                {
+                    ident = ident,
+                    filter = filter,
+                    flags = flags,
+                    fflags = fflags,
+                    data = data,
+                    udata = udata
+                };
+
+                var rc = kevent(kq, _singleEvent, 1, null, 0, _zeroTimeoutPtr);
+                if (rc != 0)
+                {
+                    throw new Exception($"kqueue failed to {flags} with error code {Marshal.GetLastWin32Error()}");
+                }
+
+                if (_singleEvent[0].flags.HasFlag(kqueue_flags.ERROR))
+                {
+                    throw new IOException($"kqueue failed to {flags} with error {_singleEvent[0].data}");
+                }
+
+                return rc;
+            }
+        }
+
+        private readonly int _kqueueHndle;
+
+        public KQueuePollGroup()
+        {
+            _kqueueHndle = BSD.kqueue();
+
+            if (_kqueueHndle == 0)
+            {
+                throw new Exception("Unable to initialize poll group");
+            }
+        }
+
+        public void Dispose()
+        {
+            BSD.close(_kqueueHndle);
+            Marshal.FreeHGlobal(_zeroTimeoutPtr);
+        }
+
+        public void Add(NetState state)
+        {
+            var rc = BSD.kevent(
+                _kqueueHndle,
+                state.Connection.Handle,
+                kqueue_filter.READ | kqueue_filter.WRITE,
+                kqueue_flags.ADD | kqueue_flags.CLEAR,
+                udata: (IntPtr)state._handle
+            );
+
+            if (rc != 0)
+            {
+                throw new Exception($"kevent failed with error code {Marshal.GetLastWin32Error()}");
+            }
+        }
+
+        public void Remove(NetState state)
+        {
+            var rc = BSD.kevent(
+                _kqueueHndle,
+                state.Connection.Handle,
+                kqueue_filter.READ | kqueue_filter.WRITE,
+                kqueue_flags.DELETE,
+                udata: (IntPtr)state._handle
+            );
+
+            if (rc != 0)
+            {
+                throw new Exception($"kevent failed with error code {Marshal.GetLastWin32Error()}");
+            }
+        }
+
+        private kevent[] _events = new kevent[2048];
+
+        public int Poll(ref NetState[] states)
+        {
+            if (states.Length > _events.Length)
+            {
+                var newLength = Math.Max(states.Length, _events.Length + (_events.Length >> 2));
+                _events = new kevent[newLength];
+            }
+
+            var rc = BSD.kevent(_kqueueHndle, null, 0, _events, _events.Length, _zeroTimeoutPtr);
+
+            if (rc <= 0)
+            {
+                return rc;
+            }
+
+            int count = 0;
+
+            for (int i = 0; i < rc; i++)
+            {
+                if (((GCHandle)_events[i].udata).Target is not NetState state)
+                {
+                    continue;
+                }
+
+                states[count++] = state;
+            }
+
+            return count;
+        }
+    }
+}

Projects/Server/Network/PollGroup/PollGroup.cs

@@ -0,0 +1,39 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2021 - ModernUO Development Team                       *
+ * Email: hi@modernuo.com                                                *
+ * File: PollGroup.cs                                                    *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+using System;
+
+namespace Server.Network
+{
+    public interface IPollGroup : IDisposable
+    {
+        void Add(NetState state);
+        void Remove(NetState state);
+        int Poll(ref NetState[] states);
+    }
+
+    public static class PollGroup
+    {
+        public static IPollGroup Create()
+        {
+            if (Core.IsBSD)
+            {
+                return new KQueuePollGroup();
+            }
+
+            return new EPollGroup();
+        }
+    }
+}

Projects/Server/Network/TcpServer.cs

@@ -84,6 +84,14 @@ namespace Server.Network
             Listeners = listeners.ToArray();
         }
 
+        public static void Shutdown()
+        {
+            foreach (var listener in Listeners)
+            {
+                listener.Server.Close();
+            }
+        }
+
         public static IEnumerable<IPEndPoint> GetListeningAddresses(IPEndPoint ipep) =>
             NetworkInterface.GetAllNetworkInterfaces().SelectMany(adapter =>
                 adapter.GetIPProperties().UnicastAddresses
@@ -93,12 +101,19 @@ namespace Server.Network
 
         public static TcpListener CreateListener(IPEndPoint ipep)
         {
-            var listener = new TcpListener(ipep);
+            var listener = new TcpListener(ipep)
-            listener.Server.ExclusiveAddressUse = false;
+            {
+                Server =
+                {
+                    LingerState = new LingerOption(false, 0),
+                    ExclusiveAddressUse = true,
+                    NoDelay = true
+                }
+            };
 
             try
             {
-                listener.Start(8);
+                listener.Start(32);
                 return listener;
             }
             catch (SocketException se)
@@ -130,7 +145,6 @@ namespace Server.Network
             {
                 Instances.Add(ns);
                 ns.LogInfo("Connected. [{0} Online]", Instances.Count);
-                ns.Start();
             }
         }
 

Projects/Server/Server.csproj

@@ -31,6 +31,12 @@
         <Delete Files="..\..\Distribution\$(AssemblyName).runtimeconfig.dev.json" ContinueOnError="true" />
         <Delete Files="..\..\Distribution\$(AssemblyName).runtimeconfig.json" ContinueOnError="true" />
     </Target>
+    <ItemGroup Condition="'$(IsWindows)'=='true' OR '$(IsWindows)'==''">
+        <Content Include="wepoll.dll">
+            <CopyToPublishDirectory>Always</CopyToPublishDirectory>
+            <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </Content>
+    </ItemGroup>
     <ItemGroup>
         <PackageReference Include="Microsoft.Toolkit.HighPerformance" Version="7.0.2" />
         <PackageReference Include="Zlib.Bindings" Version="1.5.0" />
@@ -46,7 +52,4 @@
     <ItemGroup>
         <AdditionalFiles Include="Migrations/*.v*.json" />
     </ItemGroup>
-    <ItemGroup>
-      <Compile Remove="Serialization\Attributes\SerializableFieldSaveValueDefaultAttribute.cs" />
-    </ItemGroup>
 </Project>

Projects/UOContent/Compression/ZstdArchive.cs

@@ -23,7 +23,7 @@ namespace Server.Compression
                         StartInfo = new ProcessStartInfo
                         {
                             FileName = _pathToZstd,
-                            Arguments = $"--no-progress -d \"{fileNamePath}\" -o \"${tempTarArchive}\""
+                            Arguments = $"-q -d \"{fileNamePath}\" -o \"${tempTarArchive}\""
                         }
                     };
 
@@ -70,7 +70,7 @@ namespace Server.Compression
                         StartInfo = new ProcessStartInfo
                         {
                             FileName = Path.Combine(_pathToZstd, "zstd.exe"),
-                            Arguments = $"--no-progress -10 \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\""
+                            Arguments = $"-q -10 \"{tempTarArchive}\" -o \"{destinationArchiveFileName}\""
                         }
                     };
 

THIRD-PARTY-NOTICES

@@ -84,3 +84,34 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 IN THE SOFTWARE.
+
+License notice for wepoll
+---------------------------
+wepoll - epoll for Windows
+https://github.com/piscisaureus/wepoll
+
+Copyright 2012-2020, Bert Belder <bertbelder@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

version.json

@@ -1,4 +1,4 @@
 {
   "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json",
-  "version": "0.8.2"
+  "version": "0.8.3"
 }