From 76fddcbccd294642261c9d39000c91ea243b54a9 Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Tue, 22 Mar 2022 09:58:24 -0700 Subject: [PATCH] feat: Adds a single threaded array pool (#967) ## Added Feature Adds a single threaded array pool that works exactly the same as `ArrayPool.Shared`. The `STArrayPool.Shared` can only be used on a single thread, the main game thread of the server. Note: Unlike the built-in array pool, there is no hook into the _GC Gen 2_. This means to relieve potentially high memory pressure, `ArrayPool.Shared.Trim()` must be called. The pool will only release arrays _after two successive calls within 10 seconds or longer_. If the server is at less than 70% total memory usage, or the server is not going to use this pool for something egregious, then don't bother ever calling Trim(). ## Changes - [X] Fixes ArrayPool calls that should be cleared due to references. - [X] Benchmarks against ArrayPool with 4+ rented arrays deep of the same length. - [x] Unit tests --- .../Collections/BenchmarkPooledRefQueue.cs | 95 +++++ .../Collections/BenchmarkSTArray.cs | 92 +++++ Projects/Benchmarks/Program.cs | 4 +- .../Tests/Buffers/STArrayPoolTests.cs | 71 ++++ Projects/Server/Collections/PooledRefQueue.cs | 24 +- Projects/Server/Collections/STArrayPool.cs | 324 ++++++++++++++++++ Projects/Server/Maps/Map.cs | 2 +- .../Monsters/Misc/Melee/BladeSpirits.cs | 2 +- .../Monsters/Misc/Melee/EnergyVortex.cs | 2 +- 9 files changed, 604 insertions(+), 12 deletions(-) create mode 100644 Projects/Benchmarks/Benchmarks/Collections/BenchmarkPooledRefQueue.cs create mode 100644 Projects/Benchmarks/Benchmarks/Collections/BenchmarkSTArray.cs create mode 100644 Projects/Server.Tests/Tests/Buffers/STArrayPoolTests.cs create mode 100644 Projects/Server/Collections/STArrayPool.cs diff --git a/Projects/Benchmarks/Benchmarks/Collections/BenchmarkPooledRefQueue.cs b/Projects/Benchmarks/Benchmarks/Collections/BenchmarkPooledRefQueue.cs new file mode 100644 index 000000000..f72dcd7af --- /dev/null +++ b/Projects/Benchmarks/Benchmarks/Collections/BenchmarkPooledRefQueue.cs @@ -0,0 +1,95 @@ +using System.Buffers; +using System.Collections.Generic; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Jobs; +using Server.Buffers; +using Server.Collections; + +namespace Benchmarks +{ + [MemoryDiagnoser] + [SimpleJob(RuntimeMoniker.Net60)] + public class BenchmarkPooledRefQueue + { + [GlobalSetup] + public void Setup() + { + // Allocate + var arrays = new long[16][]; + for (var i = 0; i < 16; i++) + { + arrays[i] = ArrayPool.Shared.Rent(64); + } + + var stArrays = new long[16][]; + for (var i = 0; i < 16; i++) + { + stArrays[i] = STArrayPool.Shared.Rent(64); + } + + for (var i = 0; i < 16; i++) + { + ArrayPool.Shared.Return(arrays[i]); + } + + for (var i = 0; i < 16; i++) + { + STArrayPool.Shared.Return(stArrays[i]); + } + } + + [Benchmark] + public void UseQueue() + { + for (var i = 0; i < 8; i++) + { + var queue = new Queue(); + for (var j = 0; j < 32; j++) + { + queue.Enqueue(j); + } + + for (var j = 0; j < 32; j++) + { + var num = queue.Dequeue(); + } + } + } + + [Benchmark] + public void UsePooledRefQueue() + { + for (var i = 0; i < 8; i++) + { + using var queue = PooledRefQueue.Create(); + for (var j = 0; j < 32; j++) + { + queue.Enqueue(j); + } + + for (var j = 0; j < 32; j++) + { + var num = queue.Dequeue(); + } + } + } + + [Benchmark] + public void UsePooledRefQueueMT() + { + for (var i = 0; i < 8; i++) + { + using var queue = PooledRefQueue.CreateMT(); + for (var j = 0; j < 32; j++) + { + queue.Enqueue(j); + } + + for (var j = 0; j < 32; j++) + { + var num = queue.Dequeue(); + } + } + } + } +} diff --git a/Projects/Benchmarks/Benchmarks/Collections/BenchmarkSTArray.cs b/Projects/Benchmarks/Benchmarks/Collections/BenchmarkSTArray.cs new file mode 100644 index 000000000..78be46dd9 --- /dev/null +++ b/Projects/Benchmarks/Benchmarks/Collections/BenchmarkSTArray.cs @@ -0,0 +1,92 @@ +using System.Buffers; +using System.Collections.Generic; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Jobs; +using Server.Buffers; + +namespace Benchmarks +{ + [MemoryDiagnoser] + [SimpleJob(RuntimeMoniker.Net60)] + public class BenchmarkSTArray + { + private static long[][] arrays = new long[16][]; + private static long[][] stArrays = new long[16][]; + private static long[][] newArrays = new long[16][]; + private static Queue[] newQueue = new Queue[16]; + + [GlobalSetup] + public void Setup() + { + // Allocate + arrays = new long[16][]; + for (var i = 0; i < 16; i++) + { + arrays[i] = ArrayPool.Shared.Rent(64); + } + + stArrays = new long[16][]; + for (var i = 0; i < 16; i++) + { + stArrays[i] = STArrayPool.Shared.Rent(64); + } + + for (var i = 0; i < 16; i++) + { + ArrayPool.Shared.Return(arrays[i]); + } + + for (var i = 0; i < 16; i++) + { + STArrayPool.Shared.Return(stArrays[i]); + } + } + + [Benchmark] + public void ArrayPool() + { + for (var i = 0; i < 8; i++) + { + arrays[i] = ArrayPool.Shared.Rent(64); + } + + for (var i = 0; i < 8; i++) + { + ArrayPool.Shared.Return(arrays[i], true); + } + } + + [Benchmark] + public void STArrayPool() + { + for (var i = 0; i < 8; i++) + { + arrays[i] = STArrayPool.Shared.Rent(64); + } + + for (var i = 0; i < 8; i++) + { + STArrayPool.Shared.Return(arrays[i], true); + } + } + + [Benchmark] + public void NewArray() + { + for (var i = 0; i < 8; i++) + { + newArrays[i] = new long[64]; + } + } + + [Benchmark] + public void NewQueue() + { + for (var i = 0; i < 8; i++) + { + newQueue[i] = new Queue(); + newQueue[i].EnsureCapacity(64); + } + } + } +} diff --git a/Projects/Benchmarks/Program.cs b/Projects/Benchmarks/Program.cs index e86c18d19..286664e06 100644 --- a/Projects/Benchmarks/Program.cs +++ b/Projects/Benchmarks/Program.cs @@ -26,7 +26,9 @@ namespace Benchmarks //var mapMobilesSelectors = BenchmarkRunner.Run(); //var mapMultiTilesSelectors = BenchmarkRunner.Run(); //var mapMultiSelectors = BenchmarkRunner.Run(); - var mapItemsSelectors = BenchmarkRunner.Run(); + // var mapItemsSelectors = BenchmarkRunner.Run(); + // var stArray = BenchmarkRunner.Run(); + var pooledRefQueue = BenchmarkRunner.Run(); } } } diff --git a/Projects/Server.Tests/Tests/Buffers/STArrayPoolTests.cs b/Projects/Server.Tests/Tests/Buffers/STArrayPoolTests.cs new file mode 100644 index 000000000..8b6ea0551 --- /dev/null +++ b/Projects/Server.Tests/Tests/Buffers/STArrayPoolTests.cs @@ -0,0 +1,71 @@ +using System; +using Server.Buffers; +using Xunit; + +namespace Server.Tests.Tests.Buffers; + +public class STArrayPoolTests +{ + [Theory] + [InlineData(0, 0)] + [InlineData(2, 16)] + [InlineData(56, 64)] + [InlineData(120, 128)] + [InlineData(65535, 65536)] + [InlineData(1024 * 1024 * 15, 1024 * 1024 * 16)] + public void ValidMinimumLengths(int requestedLength, int expectedLength) + { + var arr = STArrayPool.Shared.Rent(requestedLength); + Assert.Equal(expectedLength, arr.Length); + } + + [Fact] + public void NegativeLengthThrows() + { + Assert.Throws( + () => + { + var arr = STArrayPool.Shared.Rent(-1); + } + ); + } + + [Fact] + public void CachesOnlyUpToCPUCountPerBucket() + { + STArrayPool.Shared.ResetForTesting(); + + var cores = Environment.ProcessorCount; + var arrays1 = new byte[cores * 8 + 2][]; // 1 for the cache, and 8 * CPU for the stacks + var weakReferences1 = new WeakReference[cores * 8 + 2]; + + var arrays2 = new byte[cores * 8 + 2][]; // 1 for the cache, and 8 * CPU for the stacks + var weakReferences2 = new WeakReference[cores * 8 + 2]; + + for (var i = 0; i < arrays1.Length; i++) + { + arrays1[i] = STArrayPool.Shared.Rent(32); + weakReferences1[i] = new WeakReference(arrays1[i]); + + arrays2[i] = STArrayPool.Shared.Rent(64); + weakReferences2[i] = new WeakReference(arrays1[i]); + } + + for (var i = 0; i < arrays1.Length; i++) + { + STArrayPool.Shared.Return(arrays1[i]); + arrays1[i] = null; + + STArrayPool.Shared.Return(arrays2[i]); + arrays2[i] = null; + } + + GC.Collect(); + for (var i = 0; i < weakReferences1.Length; i++) + { + // When the last one is returned, the one right before it is dropped. + Assert.Equal(i != weakReferences1.Length - 2, weakReferences1[i].IsAlive); + Assert.Equal(i != weakReferences2.Length - 2, weakReferences2[i].IsAlive); + } + } +} diff --git a/Projects/Server/Collections/PooledRefQueue.cs b/Projects/Server/Collections/PooledRefQueue.cs index 9c8ea81b0..3959f0631 100644 --- a/Projects/Server/Collections/PooledRefQueue.cs +++ b/Projects/Server/Collections/PooledRefQueue.cs @@ -6,6 +6,7 @@ using System.Buffers; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using Server.Buffers; namespace Server.Collections { @@ -19,20 +20,25 @@ namespace Server.Collections private int _head; // The index from which to dequeue if the queue isn't empty. private int _tail; // The index at which to enqueue if the queue isn't full. private int _size; // Number of elements. + private bool _mt; private int _version; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static PooledRefQueue Create(int capacity = 32) => new(capacity); + public static PooledRefQueue Create(int capacity = 32, bool mt = false) => new(capacity, mt); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static PooledRefQueue CreateMT(int capacity = 32) => new(capacity, true); // Creates a queue with room for capacity objects. The default grow factor // is used. - public PooledRefQueue(int capacity) + public PooledRefQueue(int capacity, bool mt = false) { + _mt = mt; _array = capacity switch { < 0 => throw new ArgumentOutOfRangeException(nameof(capacity), capacity, CollectionThrowStrings.ArgumentOutOfRange_NeedNonNegNum), 0 => Array.Empty(), - _ => ArrayPool.Shared.Rent(capacity) + _ => (mt ? ArrayPool.Shared : STArrayPool.Shared).Rent(capacity) }; _head = 0; @@ -254,14 +260,14 @@ namespace Server.Collections return arr; } - public T[] ToPooledArray() + public T[] ToPooledArray(bool mt = false) { if (_size == 0) { return Array.Empty(); } - T[] arr = ArrayPool.Shared.Rent(_size); + T[] arr = (mt ? ArrayPool.Shared : STArrayPool.Shared).Rent(_size); if (_head < _tail) { @@ -280,7 +286,7 @@ namespace Server.Collections // must be >= _size. private void SetCapacity(int capacity) { - T[] newarray = ArrayPool.Shared.Rent(capacity); + T[] newarray = (_mt ? ArrayPool.Shared : STArrayPool.Shared).Rent(capacity); if (_size > 0) { if (_head < _tail) @@ -296,7 +302,8 @@ namespace Server.Collections if (_array.Length > 0) { - ArrayPool.Shared.Return(_array, true); + Clear(); + (_mt ? ArrayPool.Shared : STArrayPool.Shared).Return(_array); } _array = newarray; @@ -377,7 +384,8 @@ namespace Server.Collections var array = _array; if (array.Length > 0) { - ArrayPool.Shared.Return(array, true); + Clear(); + (_mt ? ArrayPool.Shared : STArrayPool.Shared).Return(array); } this = default; diff --git a/Projects/Server/Collections/STArrayPool.cs b/Projects/Server/Collections/STArrayPool.cs new file mode 100644 index 000000000..72ddca91b --- /dev/null +++ b/Projects/Server/Collections/STArrayPool.cs @@ -0,0 +1,324 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace Server.Buffers; + +/** + * Adaptation of the ArrayPool.Shared (TlsOverPerCoreLockedStacksArrayPool) for single threaded *unsafe* usage. + */ +public class STArrayPool : ArrayPool +{ + private const int StackArraySize = 8; + private const int BucketCount = 27; // SelectBucketIndex(1024 * 1024 * 1024 + 1) + private static readonly STArrayPool _shared = new(); + + public static STArrayPool Shared => _shared; + + private static STArray[] _cacheBuckets; + private STArrayStack[] _buckets = new STArrayStack[BucketCount]; + + private STArrayPool() {} + + public override T[] Rent(int minimumLength) + { + T[] buffer; + + var bucketIndex = SelectBucketIndex(minimumLength); + var cachedBuckets = _cacheBuckets; + if (cachedBuckets is not null && (uint)bucketIndex < (uint)cachedBuckets.Length) + { + buffer = cachedBuckets[bucketIndex].Array; + if (buffer is not null) + { + cachedBuckets[bucketIndex].Array = null; + return buffer; + } + } + + var buckets = _buckets; + if ((uint)bucketIndex < (uint)buckets.Length) + { + var b = buckets[bucketIndex]; + if (b is not null) + { + buffer = b.TryPop(); + if (buffer is not null) + { + return buffer; + } + } + + minimumLength = GetMaxSizeForBucket(bucketIndex); + } + + if (minimumLength == 0) + { + // We aren't renting. + return Array.Empty(); + } + + if (minimumLength < 0) + { + throw new ArgumentOutOfRangeException(nameof(minimumLength)); + } + + buffer = GC.AllocateUninitializedArray(minimumLength); + return buffer; + } + + public override void Return(T[] array, bool clearArray = false) + { + if (array is null) + { + throw new ArgumentNullException(nameof(array)); + } + + var bucketIndex = SelectBucketIndex(array.Length); + var cacheBuckets = _cacheBuckets ?? InitializeBuckets(); + + if ((uint)bucketIndex < (uint)_cacheBuckets!.Length) + { + if (clearArray) + { + Array.Clear(array); + } + + if (array.Length != GetMaxSizeForBucket(bucketIndex)) + { + throw new ArgumentException("Buffer is not from the pool", nameof(array)); + } + + ref var bucketArray = ref cacheBuckets[bucketIndex]; + var prev = bucketArray.Array; + bucketArray = new STArray(array); + if (prev is not null) + { + var bucket = _buckets[bucketIndex] ?? CreateBucketStack(bucketIndex); + bucket.TryPush(prev); + } + } + } + + public void ResetForTesting() + { + if (Core.IsRunningFromXUnit) + { + _cacheBuckets = null; + _buckets = new STArrayStack[BucketCount]; + } + } + + public bool Trim() + { + var ticks = Core.TickCount; + var pressure = GetMemoryPressure(); + + var buckets = _buckets; + for (var i = 0; i < buckets.Length; i++) + { + buckets[i]?.Trim(ticks, pressure, GetMaxSizeForBucket(i)); + } + + // Under high pressure, release all cached buckets + if (pressure == MemoryPressure.High) + { + Array.Clear(_cacheBuckets); + } + else + { + uint threshold = pressure switch + { + MemoryPressure.Medium => 10000, + _ => 30000, + }; + + var cacheBuckets = _cacheBuckets; + for (var i = 0; i < cacheBuckets.Length; i++) + { + ref var b = ref cacheBuckets[i]; + + if (b.Array is null) + { + continue; + } + + var lastSeen = b.Ticks; + if (lastSeen == 0) + { + b.Ticks = ticks; + } + else if (ticks - lastSeen >= threshold) + { + b.Array = null; + } + } + } + + return true; + } + + private STArrayStack CreateBucketStack(int bucketIndex) + { + return _buckets[bucketIndex] = new STArrayStack(); + } + + private STArray[] InitializeBuckets() + { + Debug.Assert(_cacheBuckets is null, $"Non-null {nameof(_cacheBuckets)}"); + var buckets = new STArray[BucketCount]; + return _cacheBuckets = buckets; + } + + // Buffers are bucketed so that a request between 2^(n-1) + 1 and 2^n is given a buffer of 2^n + // Bucket index is log2(bufferSize - 1) with the exception that buffers between 1 and 16 bytes + // are combined, and the index is slid down by 3 to compensate. + // Zero is a valid bufferSize, and it is assigned the highest bucket index so that zero-length + // buffers are not retained by the pool. The pool will return the Array.Empty singleton for these. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int SelectBucketIndex(int bufferSize) => BitOperations.Log2((uint)bufferSize - 1 | 15) - 3; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int GetMaxSizeForBucket(int binIndex) + { + int maxSize = 16 << binIndex; + Debug.Assert(maxSize >= 0); + return maxSize; + } + + internal enum MemoryPressure + { + Low, + Medium, + High + } + + internal static MemoryPressure GetMemoryPressure() + { + GCMemoryInfo memoryInfo = GC.GetGCMemoryInfo(); + + if (memoryInfo.MemoryLoadBytes >= memoryInfo.HighMemoryLoadThresholdBytes * 0.90) + { + return MemoryPressure.High; + } + + if (memoryInfo.MemoryLoadBytes >= memoryInfo.HighMemoryLoadThresholdBytes * 0.70) + { + return MemoryPressure.Medium; + } + + return MemoryPressure.Low; + } + + private sealed class STArrayStack + { + // Maximum buffers we will store in our stack + private readonly T[][] _arrays = new T[StackArraySize * Environment.ProcessorCount][]; + private int _count; + private long _ticks; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryPush(T[] array) + { + var arrays = _arrays; + var count = _count; + if ((uint)count < (uint)_arrays.Length) + { + arrays[count] = array; + _count = count + 1; + return true; + } + + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T[] TryPop() + { + var arrays = _arrays; + var count = _count - 1; + if ((uint)count < (uint)arrays.Length) + { + var arr = arrays[count]; + arrays[count] = null; + _count = count; + return arr; + } + + return null; + } + + public void Trim(long now, MemoryPressure pressure, int bucketSize) + { + if (_count == 0) + { + return; + } + + // 10 seconds under high pressure, otherwise 60 seconds + var threshold = pressure == MemoryPressure.High ? 10000 : 60000; + + if (_ticks == 0) + { + _ticks = now; + return; + } + + if (now - _ticks <= threshold) + { + return; + } + + int trimCount = 1; + switch (pressure) + { + case MemoryPressure.Medium: + { + trimCount = 2; + break; + } + case MemoryPressure.High: + { + if (bucketSize > 16384) + { + trimCount++; + } + + var size = Unsafe.SizeOf(); + if (size > 32) + { + trimCount += 2; + } + else if (size > 16) + { + trimCount++; + } + + break; + } + } + + while (_count > 0 && trimCount-- > 0) + { + _arrays[--_count] = null; + } + } + } + + private struct STArray + { + public T[] Array; + public long Ticks; + + public STArray(T[] array) + { + Array = array; + Ticks = 0; + } + } +} diff --git a/Projects/Server/Maps/Map.cs b/Projects/Server/Maps/Map.cs index a6fcad289..9c93f772d 100644 --- a/Projects/Server/Maps/Map.cs +++ b/Projects/Server/Maps/Map.cs @@ -691,7 +691,7 @@ namespace Server } } - ArrayPool.Shared.Return(items); + ArrayPool.Shared.Return(items, true); } /* This could probably be re-implemented if necessary (perhaps via an ITile interface?). diff --git a/Projects/UOContent/Mobiles/Monsters/Misc/Melee/BladeSpirits.cs b/Projects/UOContent/Mobiles/Monsters/Misc/Melee/BladeSpirits.cs index 374ae412e..8f326bc5f 100644 --- a/Projects/UOContent/Mobiles/Monsters/Misc/Melee/BladeSpirits.cs +++ b/Projects/UOContent/Mobiles/Monsters/Misc/Melee/BladeSpirits.cs @@ -94,7 +94,7 @@ namespace Server.Mobiles Dispel(mobs[amount--]); } - ArrayPool.Shared.Return(mobs); + ArrayPool.Shared.Return(mobs, true); } } diff --git a/Projects/UOContent/Mobiles/Monsters/Misc/Melee/EnergyVortex.cs b/Projects/UOContent/Mobiles/Monsters/Misc/Melee/EnergyVortex.cs index 145b85b6d..66f808bcc 100644 --- a/Projects/UOContent/Mobiles/Monsters/Misc/Melee/EnergyVortex.cs +++ b/Projects/UOContent/Mobiles/Monsters/Misc/Melee/EnergyVortex.cs @@ -100,7 +100,7 @@ namespace Server.Mobiles Dispel(mobs[amount--]); } - ArrayPool.Shared.Return(mobs); + ArrayPool.Shared.Return(mobs, true); } }