Replacing Networking (#271)

- [X] Removing Kestrel & Libuv
- [X] Cleaning up NetState
- [X] Removing System.IO.Pipelines
- [X] Cleaning up packet reading
- [X] Adds a maximum of 5000 sockets (configurable) to prevent OOM
- [X] Replaces the AsyncState with a thread-safe wrapped boolean called NetworkState
- [X] Removes Parallel.ForEach (no perf gain)
- [X] Removes custom houses compression on another thread
- [X] Test high load scenarios

Bumps release version

Projects/Server.Tests/Network/PacketReaderTests.cs

@@ -0,0 +1,101 @@
+using System;
+using System.IO;
+using System.Text;
+using Server.Network;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Server.Tests.Network
+{
+    public class PacketReaderTests
+    {
+        private (Encoding, Type) GetEncoding(string value) =>
+            value.ToUpper() switch
+            {
+                "UTF8"      => (Utility.UTF8, typeof(byte)),
+                "UNICODELE" => (Utility.UnicodeLE, typeof(char)),
+                "UNICODE"   => (Utility.Unicode, typeof(char)),
+                _           => (Encoding.ASCII, typeof(byte))
+            };
+
+        private ITestOutputHelper _outputHelper;
+
+        public PacketReaderTests(ITestOutputHelper outputHelper) => _outputHelper = outputHelper;
+
+        [Theory]
+        // First only, beginning
+        [InlineData("Test String", "ASCII", false, -1, 1024, 1024, 0)]
+        [InlineData("Test String", "UTF8", false, -1, 1024, 1024, 0)]
+        [InlineData("Test String", "Unicode", false, -1, 1024, 1024, 0)]
+        [InlineData("Test String", "UnicodeLE", false, -1, 1024, 1024, 0)]
+
+        // Second only
+        [InlineData("Test String", "ASCII", false, -1, 1024, 1024, 1030)]
+        [InlineData("Test String", "UTF8", false, -1, 1024, 1024, 1030)]
+        [InlineData("Test String", "Unicode", false, -1, 1024, 1024, 1030)]
+        [InlineData("Test String", "UnicodeLE", false, -1, 1024, 1024, 1030)]
+        public void TestReadString(
+            string value,
+            string encodingStr,
+            bool isSafe,
+            int fixedLength,
+            int firstSize,
+            int secondSize,
+            int offset
+        )
+        {
+            var (encoding, byteType) = GetEncoding(encodingStr);
+            var bytes = encoding.GetBytes(value);
+            Span<byte> expectedBytes = bytes.AsSpan(0, fixedLength > -1 ? Math.Min(fixedLength, bytes.Length) : bytes.Length);
+
+            if (offset + expectedBytes.Length > firstSize + secondSize)
+            {
+                throw new ArgumentException("Test failed due to bad assumptions. Out of memory exception would be thrown");
+            }
+
+            Span<byte> first = stackalloc byte[firstSize];
+            first.Clear(); // Just in case
+
+            int bytesWrittenFirst = 0;
+
+            if (offset < first.Length)
+            {
+                bytesWrittenFirst = Math.Min(first.Length, expectedBytes.Length);
+                expectedBytes.Slice(offset, bytesWrittenFirst).CopyTo(first);
+            }
+
+            Span<byte> second = stackalloc byte[secondSize];
+            second.Clear(); // Just in case
+
+            if (offset > first.Length || expectedBytes.Length > bytesWrittenFirst)
+            {
+                var secondOffset = Math.Max(offset - first.Length, 0);
+                var secondSlice = second.Slice(secondOffset, second.Length - secondOffset);
+                expectedBytes.Slice(bytesWrittenFirst, expectedBytes.Length - bytesWrittenFirst).CopyTo(secondSlice);
+            }
+
+            // _outputHelper.WriteLine(HexStringConverter.GetString(first));
+            // _outputHelper.WriteLine(HexStringConverter.GetString(second));
+
+            var reader = new PacketReader(first, second);
+            reader.Seek(offset, SeekOrigin.Begin);
+
+            _outputHelper.WriteLine(reader.Position.ToString());
+
+            var actual = byteType switch
+            {
+                var b when b == typeof(char) => reader.ReadString<char>(encoding, isSafe, fixedLength),
+                _                                  => reader.ReadString<byte>(encoding, isSafe, fixedLength)
+            };
+
+            _outputHelper.WriteLine(actual);
+
+            if (fixedLength > 0 && fixedLength < value.Length)
+            {
+                value = value.Substring(0, fixedLength);
+            }
+
+            Assert.Equal(value, actual);
+        }
+    }
+}

Projects/Server.Tests/Network/Packets/Old/Outgoing/AccountPacketTests.cs

@@ -1,11 +1,7 @@
 using System;
 using System.Buffers;
-using System.Collections.Generic;
-using System.IO.Pipelines;
 using System.Linq;
 using System.Net;
-using Microsoft.AspNetCore.Connections;
-using Microsoft.AspNetCore.Http.Features;
 using Server.Accounting;
 using Server.Network;
 using Xunit;
@@ -111,12 +107,7 @@ namespace Server.Tests.Network.Packets
 
             var account = new TestAccount(new[] { firstMobile, null, null, null, null });
 
-            var ns = new NetState(
-                new TestConnectionContext
-                {
-                    RemoteEndPoint = IPEndPoint.Parse("127.0.0.1")
-                }
-            )
+            var ns = new NetState(null)
             {
                 Account = account,
                 ProtocolChanges = protocolChanges
@@ -653,13 +644,5 @@ namespace Server.Tests.Network.Packets
 
             public int CompareTo(TestAccount other) => other == null ? 1 : Username.CompareTo(other.Username);
         }
-
-        internal class TestConnectionContext : ConnectionContext
-        {
-            public override string ConnectionId { get; set; }
-            public override IFeatureCollection Features { get; }
-            public override IDictionary<object, object> Items { get; set; }
-            public override IDuplexPipe Transport { get; set; }
-        }
     }
 }

Projects/Server.Tests/Network/Packets/Old/Outgoing/GumpPacketTests.cs

@@ -59,12 +59,7 @@ namespace Server.Tests.Network.Packets
         [Fact]
         public void TestFastGumpPacket()
         {
-            var ns = new NetState(
-                new AccountPacketTests.TestConnectionContext
-                {
-                    RemoteEndPoint = IPEndPoint.Parse("127.0.0.1")
-                }
-            );
+            var ns = new NetState(null);
 
             var gump = new ResurrectGump(2);
 
@@ -134,12 +129,7 @@ namespace Server.Tests.Network.Packets
         [Fact]
         public void TestPackedGumpPacket()
         {
-            var ns = new NetState(
-                new AccountPacketTests.TestConnectionContext
-                {
-                    RemoteEndPoint = IPEndPoint.Parse("127.0.0.1")
-                }
-            )
+            var ns = new NetState(null)
             {
                 ProtocolChanges = ProtocolChanges.Unpack
             };
@@ -194,9 +184,8 @@ namespace Server.Tests.Network.Packets
                 layoutList.Add(str);
             }
 
-            var memOwner = SlabMemoryPool.Shared.Rent(bufferLength);
-
-            var buffer = memOwner.Memory.Span;
+            var rawBuffer = ArrayPool<byte>.Shared.Rent(bufferLength);
+            Span<byte> buffer = rawBuffer;
             var bufferPos = 0;
 
             foreach (var layout in layoutList)
@@ -211,12 +200,12 @@ namespace Server.Tests.Network.Packets
 #endif
 
             expectedData.WritePacked(ref pos, buffer.Slice(0, bufferPos));
-            memOwner.Dispose();
+            ArrayPool<byte>.Shared.Return(rawBuffer);
 
             expectedData.Write(ref pos, gump.Strings.Count);
             bufferLength = gump.Strings.Sum(str => 2 + str.Length * 2);
-            memOwner = SlabMemoryPool.Shared.Rent(bufferLength);
-            buffer = memOwner.Memory.Span;
+            rawBuffer = ArrayPool<byte>.Shared.Rent(bufferLength);
+            buffer = rawBuffer;
             bufferPos = 0;
 
             foreach (var str in gump.Strings)
@@ -225,6 +214,7 @@ namespace Server.Tests.Network.Packets
             }
 
             expectedData.WritePacked(ref pos, buffer.Slice(0, bufferPos));
+            ArrayPool<byte>.Shared.Return(rawBuffer);
 
             // Length
             expectedData.Slice(1, 2).Write((ushort)pos);

Projects/Server.Tests/Network/Packets/Old/Outgoing/MobilePacketTests.cs

@@ -154,7 +154,7 @@ namespace Server.Tests.Network.Packets
             Span<byte> expectedData = stackalloc byte[14];
             var pos = 0;
 
-            expectedData.Write(ref pos, (byte)0x6E);
+            expectedData.Write(ref pos, (byte)0x6E); // Packet ID
             expectedData.Write(ref pos, mobile);
             expectedData.Write(ref pos, (ushort)action);
             expectedData.Write(ref pos, (ushort)frameCount);
@@ -240,12 +240,7 @@ namespace Server.Tests.Network.Packets
             };
             beheld.DefaultMobileInit();
 
-            var ns = new NetState(
-                new AccountPacketTests.TestConnectionContext
-                {
-                    RemoteEndPoint = IPEndPoint.Parse("127.0.0.1")
-                }
-            )
+            var ns = new NetState(null)
             {
                 ProtocolChanges = changes
             };
@@ -642,12 +637,7 @@ namespace Server.Tests.Network.Packets
             beheld.FacialHairItemID = facialHairItemId;
             beheld.FacialHairHue = facialHairHue;
 
-            var ns = new NetState(
-                new AccountPacketTests.TestConnectionContext
-                {
-                    RemoteEndPoint = IPEndPoint.Parse("127.0.0.1")
-                }
-            )
+            var ns = new NetState(null)
             {
                 ProtocolChanges = protocolChanges
             };

Projects/Server.Tests/Network/PipeTests.cs

@@ -0,0 +1,214 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Server.Network;
+using Xunit;
+
+namespace Server.Tests.Network
+{
+    public class PipeTests
+    {
+        private async void DelayedExecute(Action action)
+        {
+            await Task.Delay(5);
+
+            action();
+        }
+
+        [Fact]
+        public async void Await()
+        {
+            var pipe = new Pipe<byte>(new byte[100]);
+
+            var reader = pipe.Reader;
+            var writer = pipe.Writer;
+
+            DelayedExecute(() =>
+            {
+                // Write some data into the pipe
+                var buffer = writer.GetAvailable();
+                Assert.True(buffer.Length == 99);
+
+                buffer.CopyFrom(new byte[] { 1 });
+                buffer.CopyFrom(new byte[] { 2 });
+                buffer.CopyFrom(new byte[] { 3 });
+
+                writer.Advance(3);
+                writer.Flush();
+            });
+
+            var result = await reader;
+
+            Assert.True(result.Buffer[0].Count == 3);
+        }
+
+        private bool _signal;
+
+        private void Consumer(object state)
+        {
+            var reader = ((Pipe<byte>)state).Reader;
+
+            int count = 0;
+            byte expected_value = 0xFA;
+
+            while (count < 0x8000000)
+            {
+                var result = reader.TryRead();
+
+                for (int i = 0; i < result.Buffer[0].Count; i++)
+                {
+                    Assert.True(result.Buffer[0][i] == expected_value);
+                    count++;
+
+                    if (count == 0x1000)
+                    {
+                        expected_value = 0xAC;
+                    }
+                }
+
+                for (int i = 0; i < result.Buffer[1].Count; i++)
+                {
+                    Assert.True(result.Buffer[1][i] == expected_value);
+                    count++;
+
+                    if (count == 0x1000)
+                    {
+                        expected_value = 0xAC;
+                    }
+                }
+
+                reader.Advance((uint)result.Length);
+            }
+
+            _signal = true;
+        }
+
+        [Fact]
+        public async void Threading()
+        {
+            var pipe = new Pipe<byte>(new byte[0x1001]);
+
+            ThreadPool.UnsafeQueueUserWorkItem(Consumer, pipe);
+
+            var writer = pipe.Writer;
+
+            int count = 0;
+            byte expected_value = 0xFA;
+
+            while (count < 0x8000000)
+            {
+                var result = writer.GetAvailable();
+
+                if (result.Length < 16)
+                {
+                    continue;
+                }
+
+                result.CopyFrom(new[] { expected_value, expected_value, expected_value, expected_value, expected_value, expected_value, expected_value, expected_value,
+                    expected_value, expected_value, expected_value, expected_value, expected_value, expected_value, expected_value, expected_value });
+
+                writer.Advance(16);
+                count += 16;
+
+                if (count == 0x1000)
+                {
+                    expected_value = 0xAC;
+                }
+            }
+
+            while (_signal == false) { }
+
+            _signal = false;
+        }
+
+        [Fact]
+        public void Wrap()
+        {
+            var pipe = new Pipe<byte>(new byte[10]);
+
+            var reader = pipe.Reader;
+            var writer = pipe.Writer;
+
+            var result = writer.GetAvailable();
+            Assert.True(result.Length == 9);
+            Assert.True(reader.GetRemaining() == 0);
+            result = reader.TryRead();
+            Assert.True(result.Length == 0);
+
+            writer.Advance(7);
+            result = writer.GetAvailable();
+            Assert.True(result.Length == 2);
+            Assert.True(reader.GetRemaining() == 7);
+            result = reader.TryRead();
+            Assert.True(result.Length == 7);
+
+            reader.Advance(4);
+            result = writer.GetAvailable();
+            Assert.True(result.Length == 6);
+            Assert.True(reader.GetRemaining() == 3);
+            result = reader.TryRead();
+            Assert.True(result.Length == 3);
+
+            writer.Advance(3);
+            result = writer.GetAvailable();
+            Assert.True(result.Length == 3);
+            Assert.True(reader.GetRemaining() == 6);
+            result = reader.TryRead();
+            Assert.True(result.Length == 6);
+
+        }
+
+        [Fact]
+        public void Match()
+        {
+            var pipe = new Pipe<byte>(new byte[10]);
+
+            var reader = pipe.Reader;
+            var writer = pipe.Writer;
+
+            for (uint i = 0; i < 9; i++)
+            {
+                writer.Advance(i);
+                writer.Flush();
+
+                Assert.True(reader.GetRemaining() == i);
+                reader.Advance(i);
+            }
+        }
+
+        [Fact]
+        public void Sequence()
+        {
+            var pipe = new Pipe<byte>(new byte[10]);
+
+            var reader = pipe.Reader;
+            var writer = pipe.Writer;
+
+            var buffer = writer.GetAvailable();
+            Assert.True(buffer.Length == 9);
+
+            buffer.CopyFrom(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 });
+
+            writer.Advance(9);
+            writer.Flush();
+
+            Assert.True(reader.GetRemaining() == 9);
+
+            buffer = reader.TryRead();
+
+            for (int i = 0; i < 9; i++)
+            {
+                Assert.True(buffer.Buffer[0][i] == i);
+            }
+
+            reader.Advance(4);
+            buffer = reader.TryRead();
+            Assert.True(buffer.Length == 5);
+            Assert.True(buffer.Buffer[0][0] == 4);
+            Assert.True(buffer.Buffer[0][1] == 5);
+            Assert.True(buffer.Buffer[0][2] == 6);
+            Assert.True(buffer.Buffer[0][3] == 7);
+            Assert.True(buffer.Buffer[0][4] == 8);
+        }
+    }
+}