From 87b63b38a53d61eb7287d7dd1b2525868cd14be4 Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Tue, 10 May 2022 18:50:23 -0700 Subject: [PATCH] fix: Eliminates string allocations while writing gump packets (#1017) * Introduces `RawInterpolatedStringHandler` which is exactly the same as `DefaultInterpolatedStringHandler` except it _unsafely exposes_ it's `ReadOnlySpan` buffer. This is useful for writing the string's data without actually building the string. * Uses this new string interpolation handler in `SpanWriter` to eliminate intermediate strings built. This is immensely useful in eliminating string allocations in writing Gump packets. --- .../Server/Buffers/CircularBufferReader.cs | 16 +- .../Buffers/PooledArraySpanFormattable.cs | 3 +- .../Buffers/RawInterpolatedStringHandler.cs | 600 ++++++++++++++++++ Projects/Server/Buffers/SpanReader.cs | 8 +- Projects/Server/Buffers/SpanWriter.cs | 76 ++- .../Server/Localization/LocalizationEntry.cs | 12 +- Projects/Server/Text/TextEncoding.cs | 2 +- 7 files changed, 674 insertions(+), 43 deletions(-) create mode 100644 Projects/Server/Buffers/RawInterpolatedStringHandler.cs diff --git a/Projects/Server/Buffers/CircularBufferReader.cs b/Projects/Server/Buffers/CircularBufferReader.cs index 40eeafc2f..aa8b9aa08 100644 --- a/Projects/Server/Buffers/CircularBufferReader.cs +++ b/Projects/Server/Buffers/CircularBufferReader.cs @@ -245,7 +245,7 @@ namespace Server.Network [MethodImpl(MethodImplOptions.AggressiveInlining)] public string ReadString(Encoding encoding, bool safeString = false, int fixedLength = -1) { - int sizeT = TextEncoding.GetByteLengthForEncoding(encoding); + int byteLength = encoding.GetByteLengthForEncoding(); bool isFixedLength = fixedLength > -1; @@ -254,7 +254,7 @@ namespace Server.Network if (isFixedLength) { - size = fixedLength * sizeT; + size = fixedLength * byteLength; if (size > Remaining) { throw new OutOfMemoryException(); @@ -262,7 +262,7 @@ namespace Server.Network } else { - size = remaining - (remaining & (sizeT - 1)); + size = remaining - (remaining & (byteLength - 1)); } ReadOnlySpan span; @@ -273,7 +273,7 @@ namespace Server.Network var firstLength = Math.Min(_first.Length - Position, size); // Find terminator - index = _first.Slice(Position, firstLength).IndexOfTerminator(sizeT); + index = _first.Slice(Position, firstLength).IndexOfTerminator(byteLength); if (index < 0) { @@ -285,7 +285,7 @@ namespace Server.Network } else { - index = _second[..remaining].IndexOfTerminator(sizeT); + index = _second[..remaining].IndexOfTerminator(byteLength); int secondLength = index < 0 ? remaining : index; int length = firstLength + secondLength; @@ -295,7 +295,7 @@ namespace Server.Network _first[Position..].CopyTo(bytes); _second[..secondLength].CopyTo(bytes[firstLength..]); - Position += length + (index >= 0 ? sizeT : 0); + Position += length + (index >= 0 ? byteLength : 0); return TextEncoding.GetString(bytes, encoding, safeString); } } @@ -306,7 +306,7 @@ namespace Server.Network { size = Math.Min(remaining, size); span = _second.Slice( Position - _first.Length, size); - index = span.IndexOfTerminator(sizeT); + index = span.IndexOfTerminator(byteLength); if (index >= 0) { @@ -318,7 +318,7 @@ namespace Server.Network } } - Position += isFixedLength ? size : index + sizeT; + Position += isFixedLength ? size : index + byteLength; return TextEncoding.GetString(span, encoding, safeString); } diff --git a/Projects/Server/Buffers/PooledArraySpanFormattable.cs b/Projects/Server/Buffers/PooledArraySpanFormattable.cs index 50e7df384..a210a7ff9 100644 --- a/Projects/Server/Buffers/PooledArraySpanFormattable.cs +++ b/Projects/Server/Buffers/PooledArraySpanFormattable.cs @@ -15,7 +15,6 @@ #nullable enable using System; -using System.Buffers; namespace Server.Buffers; @@ -64,7 +63,7 @@ public struct PooledArraySpanFormattable : ISpanFormattable, IDisposable { if (_arrayToReturnToPool != null) { - ArrayPool.Shared.Return(_arrayToReturnToPool); + STArrayPool.Shared.Return(_arrayToReturnToPool); _arrayToReturnToPool = null; } } diff --git a/Projects/Server/Buffers/RawInterpolatedStringHandler.cs b/Projects/Server/Buffers/RawInterpolatedStringHandler.cs new file mode 100644 index 000000000..20c996ead --- /dev/null +++ b/Projects/Server/Buffers/RawInterpolatedStringHandler.cs @@ -0,0 +1,600 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.CompilerServices; + +namespace Server.Buffers; + +/// Provides a handler to interpolate strings which UNSAFELY exposes it's internal character span. +[InterpolatedStringHandler] +public ref struct RawInterpolatedStringHandler +{ + // Implementation note: + // As this type lives in CompilerServices and is only intended to be targeted by the compiler, + // public APIs eschew argument validation logic in a variety of places, e.g. allowing a null input + // when one isn't expected to produce a NullReferenceException rather than an ArgumentNullException. + + /// Expected average length of formatted data used for an individual interpolation expression result. + /// + /// This is inherited from string.Format, and could be changed based on further data. + /// string.Format actually uses `format.Length + args.Length * 8`, but format.Length + /// includes the format items themselves, e.g. "{0}", and since it's rare to have double-digit + /// numbers of items, we bump the 8 up to 11 to account for the three extra characters in "{d}", + /// since the compiler-provided base length won't include the equivalent character count. + /// + private const int GuessedLengthPerHole = 11; + /// Minimum size array to rent from the pool. + /// Same as stack-allocation size used today by string.Format. + private const int MinimumArrayPoolLength = 256; + + /// Optional provider to pass to IFormattable.ToString or ISpanFormattable.TryFormat calls. + private readonly IFormatProvider? _provider; + /// Array rented from the array pool and used to back . + private char[]? _arrayToReturnToPool; + /// The span to write into. + private Span _chars; + /// Position at which to write the next character. + private int _pos; + /// Whether provides an ICustomFormatter. + /// + /// Custom formatters are very rare. We want to support them, but it's ok if we make them more expensive + /// in order to make them as pay-for-play as possible. So, we avoid adding another reference type field + /// to reduce the size of the handler and to reduce required zero'ing, by only storing whether the provider + /// provides a formatter, rather than actually storing the formatter. This in turn means, if there is a + /// formatter, we pay for the extra interface call on each AppendFormatted that needs it. + /// + private readonly bool _hasCustomFormatter; + + /// Creates a handler used to translate an interpolated string into a . + /// The number of constant characters outside of interpolation expressions in the interpolated string. + /// The number of interpolation expressions in the interpolated string. + /// This is intended to be called only by compiler-generated code. Arguments are not validated as they'd otherwise be for members intended to be used directly. + public RawInterpolatedStringHandler(int literalLength, int formattedCount) + { + _provider = null; + _chars = _arrayToReturnToPool = STArrayPool.Shared.Rent(GetDefaultLength(literalLength, formattedCount)); + _pos = 0; + _hasCustomFormatter = false; + } + + /// Creates a handler used to translate an interpolated string into a . + /// The number of constant characters outside of interpolation expressions in the interpolated string. + /// The number of interpolation expressions in the interpolated string. + /// An object that supplies culture-specific formatting information. + /// This is intended to be called only by compiler-generated code. Arguments are not validated as they'd otherwise be for members intended to be used directly. + public RawInterpolatedStringHandler(int literalLength, int formattedCount, IFormatProvider? provider) + { + _provider = provider; + _chars = _arrayToReturnToPool = STArrayPool.Shared.Rent(GetDefaultLength(literalLength, formattedCount)); + _pos = 0; + _hasCustomFormatter = provider is not null && HasCustomFormatter(provider); + } + + /// Derives a default length with which to seed the handler. + /// The number of constant characters outside of interpolation expressions in the interpolated string. + /// The number of interpolation expressions in the interpolated string. + [MethodImpl(MethodImplOptions.AggressiveInlining)] // becomes a constant when inputs are constant + internal static int GetDefaultLength(int literalLength, int formattedCount) => + Math.Max(MinimumArrayPoolLength, literalLength + (formattedCount * GuessedLengthPerHole)); + + /// Clears the handler, returning any rented array to the pool. + [MethodImpl(MethodImplOptions.AggressiveInlining)] // used only on a few hot paths + public void Clear() + { + char[]? toReturn = _arrayToReturnToPool; + this = default; // defensive clear + if (toReturn is not null) + { + STArrayPool.Shared.Return(toReturn); + } + } + + /// Gets a span of the written characters thus far. + public ReadOnlySpan Text => _chars[.._pos]; + + /// Writes the specified string to the handler. + /// The string to write. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendLiteral(string value) + { + if (value.Length == 1) + { + Span chars = _chars; + int pos = _pos; + if ((uint)pos < (uint)chars.Length) + { + chars[pos] = value[0]; + _pos = pos + 1; + } + else + { + GrowThenCopyString(value); + } + return; + } + + AppendStringDirect(value); + } + + /// Writes the specified string to the handler. + /// The string to write. + private void AppendStringDirect(string value) + { + if (value.TryCopyTo(_chars[_pos..])) + { + _pos += value.Length; + } + else + { + GrowThenCopyString(value); + } + } + + #region AppendFormatted + // Design note: + // The compiler requires a AppendFormatted overload for anything that might be within an interpolation expression; + // if it can't find an appropriate overload, for handlers in general it'll simply fail to compile. + // (For target-typing to string where it uses DefaultInterpolatedStringHandler implicitly, it'll instead fall back to + // its other mechanisms, e.g. using string.Format. This fallback has the benefit that if we miss a case, + // interpolated strings will still work, but it has the downside that a developer generally won't know + // if the fallback is happening and they're paying more.) + // + // At a minimum, then, we would need an overload that accepts: + // (object value, int alignment = 0, string? format = null) + // Such an overload would provide the same expressiveness as string.Format. However, this has several + // shortcomings: + // - Every value type in an interpolation expression would be boxed. + // - ReadOnlySpan could not be used in interpolation expressions. + // - Every AppendFormatted call would have three arguments at the call site, bloating the IL further. + // - Every invocation would be more expensive, due to lack of specialization, every call needing to account + // for alignment and format, etc. + // + // To address that, we could just have overloads for T and ReadOnlySpan: + // (T) + // (T, int alignment) + // (T, string? format) + // (T, int alignment, string? format) + // (ReadOnlySpan) + // (ReadOnlySpan, int alignment) + // (ReadOnlySpan, string? format) + // (ReadOnlySpan, int alignment, string? format) + // but this also has shortcomings: + // - Some expressions that would have worked with an object overload will now force a fallback to string.Format + // (or fail to compile if the handler is used in places where the fallback isn't provided), because the compiler + // can't always target type to T, e.g. `b switch { true => 1, false => null }` where `b` is a bool can successfully + // be passed as an argument of type `object` but not of type `T`. + // - Reference types get no benefit from going through the generic code paths, and actually incur some overheads + // from doing so. + // - Nullable value types also pay a heavy price, in particular around interface checks that would generally evaporate + // at compile time for value types but don't (currently) if the Nullable goes through the same code paths + // (see https://github.com/dotnet/runtime/issues/50915). + // + // We could try to take a more elaborate approach for DefaultInterpolatedStringHandler, since it is the most common handler + // and we want to minimize overheads both at runtime and in IL size, e.g. have a complete set of overloads for each of: + // (T, ...) where T : struct + // (T?, ...) where T : struct + // (object, ...) + // (ReadOnlySpan, ...) + // (string, ...) + // but this also has shortcomings, most importantly: + // - If you have an unconstrained T that happens to be a value type, it'll now end up getting boxed to use the object overload. + // This also necessitates the T? overload, since nullable value types don't meet a T : struct constraint, so without those + // they'd all map to the object overloads as well. + // - Any reference type with an implicit cast to ROS will fail to compile due to ambiguities between the overloads. string + // is one such type, hence needing dedicated overloads for it that can be bound to more tightly. + // + // A middle ground we've settled on, which is likely to be the right approach for most other handlers as well, would be the set: + // (T, ...) with no constraint + // (ReadOnlySpan) and (ReadOnlySpan, int) + // (object, int alignment = 0, string? format = null) + // (string) and (string, int) + // This would address most of the concerns, at the expense of: + // - Most reference types going through the generic code paths and so being a bit more expensive. + // - Nullable types being more expensive until https://github.com/dotnet/runtime/issues/50915 is addressed. + // We could choose to add a T? where T : struct set of overloads if necessary. + // Strings don't require their own overloads here, but as they're expected to be very common and as we can + // optimize them in several ways (can copy the contents directly, don't need to do any interface checks, don't + // need to pay the shared generic overheads, etc.) we can add overloads specifically to optimize for them. + // + // Hole values are formatted according to the following policy: + // 1. If an IFormatProvider was supplied and it provides an ICustomFormatter, use ICustomFormatter.Format (even if the value is null). + // 2. If the type implements ISpanFormattable, use ISpanFormattable.TryFormat. + // 3. If the type implements IFormattable, use IFormattable.ToString. + // 4. Otherwise, use object.ToString. + // This matches the behavior of string.Format, StringBuilder.AppendFormat, etc. The only overloads for which this doesn't + // apply is ReadOnlySpan, which isn't supported by either string.Format nor StringBuilder.AppendFormat, but more + // importantly which can't be boxed to be passed to ICustomFormatter.Format. + + #region AppendFormatted T + /// Writes the specified value to the handler. + /// The value to write. + public void AppendFormatted(T value) + { + // This method could delegate to AppendFormatted with a null format, but explicitly passing + // default as the format to TryFormat helps to improve code quality in some cases when TryFormat is inlined, + // e.g. for Int32 it enables the JIT to eliminate code in the inlined method based on a length check on the format. + + // If there's a custom formatter, always use it. + if (_hasCustomFormatter) + { + AppendCustomFormatter(value, format: null); + return; + } + + // Check first for IFormattable, even though we'll prefer to use ISpanFormattable, as the latter + // requires the former. For value types, it won't matter as the type checks devolve into + // JIT-time constants. For reference types, they're more likely to implement IFormattable + // than they are to implement ISpanFormattable: if they don't implement either, we save an + // interface check over first checking for ISpanFormattable and then for IFormattable, and + // if it only implements IFormattable, we come out even: only if it implements both do we + // end up paying for an extra interface check. + string? s; + if (value is IFormattable) + { + // If the value can format itself directly into our buffer, do so. + if (value is ISpanFormattable) + { + int charsWritten; + while (!((ISpanFormattable)value).TryFormat(_chars[_pos..], out charsWritten, default, _provider)) // constrained call avoiding boxing for value types + { + Grow(); + } + + _pos += charsWritten; + return; + } + + s = ((IFormattable)value).ToString(format: null, _provider); // constrained call avoiding boxing for value types + } + else + { + s = value?.ToString(); + } + + if (s is not null) + { + AppendStringDirect(s); + } + } + /// Writes the specified value to the handler. + /// The value to write. + /// The format string. + public void AppendFormatted(T value, string? format) + { + // If there's a custom formatter, always use it. + if (_hasCustomFormatter) + { + AppendCustomFormatter(value, format); + return; + } + + // Check first for IFormattable, even though we'll prefer to use ISpanFormattable, as the latter + // requires the former. For value types, it won't matter as the type checks devolve into + // JIT-time constants. For reference types, they're more likely to implement IFormattable + // than they are to implement ISpanFormattable: if they don't implement either, we save an + // interface check over first checking for ISpanFormattable and then for IFormattable, and + // if it only implements IFormattable, we come out even: only if it implements both do we + // end up paying for an extra interface check. + string? s; + if (value is IFormattable) + { + // If the value can format itself directly into our buffer, do so. + if (value is ISpanFormattable) + { + int charsWritten; + while (!((ISpanFormattable)value).TryFormat(_chars[_pos..], out charsWritten, format, _provider)) // constrained call avoiding boxing for value types + { + Grow(); + } + + _pos += charsWritten; + return; + } + + s = ((IFormattable)value).ToString(format, _provider); // constrained call avoiding boxing for value types + } + else + { + s = value?.ToString(); + } + + if (s is not null) + { + AppendStringDirect(s); + } + } + + /// Writes the specified value to the handler. + /// The value to write. + /// Minimum number of characters that should be written for this value. If the value is negative, it indicates left-aligned and the required minimum is the absolute value. + public void AppendFormatted(T value, int alignment) + { + int startingPos = _pos; + AppendFormatted(value); + if (alignment != 0) + { + AppendOrInsertAlignmentIfNeeded(startingPos, alignment); + } + } + + /// Writes the specified value to the handler. + /// The value to write. + /// The format string. + /// Minimum number of characters that should be written for this value. If the value is negative, it indicates left-aligned and the required minimum is the absolute value. + public void AppendFormatted(T value, int alignment, string? format) + { + int startingPos = _pos; + AppendFormatted(value, format); + if (alignment != 0) + { + AppendOrInsertAlignmentIfNeeded(startingPos, alignment); + } + } + #endregion + + #region AppendFormatted ReadOnlySpan + /// Writes the specified character span to the handler. + /// The span to write. + public void AppendFormatted(ReadOnlySpan value) + { + // Fast path for when the value fits in the current buffer + if (value.TryCopyTo(_chars[_pos..])) + { + _pos += value.Length; + } + else + { + GrowThenCopySpan(value); + } + } + + /// Writes the specified string of chars to the handler. + /// The span to write. + /// Minimum number of characters that should be written for this value. If the value is negative, it indicates left-aligned and the required minimum is the absolute value. + /// The format string. + public void AppendFormatted(ReadOnlySpan value, int alignment = 0, string? format = null) + { + bool leftAlign = false; + if (alignment < 0) + { + leftAlign = true; + alignment = -alignment; + } + + int paddingRequired = alignment - value.Length; + if (paddingRequired <= 0) + { + // The value is as large or larger than the required amount of padding, + // so just write the value. + AppendFormatted(value); + return; + } + + // Write the value along with the appropriate padding. + EnsureCapacityForAdditionalChars(value.Length + paddingRequired); + if (leftAlign) + { + value.CopyTo(_chars[_pos..]); + _pos += value.Length; + _chars.Slice(_pos, paddingRequired).Fill(' '); + _pos += paddingRequired; + } + else + { + _chars.Slice(_pos, paddingRequired).Fill(' '); + _pos += paddingRequired; + value.CopyTo(_chars[_pos..]); + _pos += value.Length; + } + } + #endregion + + #region AppendFormatted string + /// Writes the specified value to the handler. + /// The value to write. + public void AppendFormatted(string? value) + { + // Fast-path for no custom formatter and a non-null string that fits in the current destination buffer. + if (!_hasCustomFormatter && value?.TryCopyTo(_chars[_pos..]) == true) + { + _pos += value.Length; + } + else + { + AppendFormattedSlow(value); + } + } + + /// Writes the specified value to the handler. + /// The value to write. + /// + /// Slow path to handle a custom formatter, potentially null value, + /// or a string that doesn't fit in the current buffer. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private void AppendFormattedSlow(string? value) + { + if (_hasCustomFormatter) + { + AppendCustomFormatter(value, format: null); + } + else if (value is not null) + { + EnsureCapacityForAdditionalChars(value.Length); + value.CopyTo(_chars[_pos..]); + _pos += value.Length; + } + } + + /// Writes the specified value to the handler. + /// The value to write. + /// Minimum number of characters that should be written for this value. If the value is negative, it indicates left-aligned and the required minimum is the absolute value. + /// The format string. + public void AppendFormatted(string? value, int alignment = 0, string? format = null) => + // Format is meaningless for strings and doesn't make sense for someone to specify. We have the overload + // simply to disambiguate between ROS and object, just in case someone does specify a format, as + // string is implicitly convertible to both. Just delegate to the T-based implementation. + AppendFormatted(value, alignment, format); + #endregion + + #region AppendFormatted object + /// Writes the specified value to the handler. + /// The value to write. + /// Minimum number of characters that should be written for this value. If the value is negative, it indicates left-aligned and the required minimum is the absolute value. + /// The format string. + public void AppendFormatted(object? value, int alignment = 0, string? format = null) => + // This overload is expected to be used rarely, only if either a) something strongly typed as object is + // formatted with both an alignment and a format, or b) the compiler is unable to target type to T. It + // exists purely to help make cases from (b) compile. Just delegate to the T-based implementation. + AppendFormatted(value, alignment, format); + #endregion + #endregion + + /// Gets whether the provider provides a custom formatter. + [MethodImpl(MethodImplOptions.AggressiveInlining)] // only used in a few hot path call sites + internal static bool HasCustomFormatter(IFormatProvider provider) + { + Debug.Assert(provider is not null); + Debug.Assert(provider is not CultureInfo || provider.GetFormat(typeof(ICustomFormatter)) is null, "Expected CultureInfo to not provide a custom formatter"); + return + provider.GetType() != typeof(CultureInfo) && // optimization to avoid GetFormat in the majority case + provider.GetFormat(typeof(ICustomFormatter)) != null; + } + + /// Formats the value using the custom formatter from the provider. + /// The value to write. + /// The format string. + [MethodImpl(MethodImplOptions.NoInlining)] + private void AppendCustomFormatter(T value, string? format) + { + // This case is very rare, but we need to handle it prior to the other checks in case + // a provider was used that supplied an ICustomFormatter which wanted to intercept the particular value. + // We do the cast here rather than in the ctor, even though this could be executed multiple times per + // formatting, to make the cast pay for play. + Debug.Assert(_hasCustomFormatter); + Debug.Assert(_provider != null); + + ICustomFormatter? formatter = (ICustomFormatter?)_provider.GetFormat(typeof(ICustomFormatter)); + Debug.Assert(formatter != null, "An incorrectly written provider said it implemented ICustomFormatter, and then didn't"); + + if (formatter?.Format(format, value, _provider) is string customFormatted) + { + AppendStringDirect(customFormatted); + } + } + + /// Handles adding any padding required for aligning a formatted value in an interpolation expression. + /// The position at which the written value started. + /// Non-zero minimum number of characters that should be written for this value. If the value is negative, it indicates left-aligned and the required minimum is the absolute value. + private void AppendOrInsertAlignmentIfNeeded(int startingPos, int alignment) + { + Debug.Assert(startingPos >= 0 && startingPos <= _pos); + Debug.Assert(alignment != 0); + + int charsWritten = _pos - startingPos; + + bool leftAlign = false; + if (alignment < 0) + { + leftAlign = true; + alignment = -alignment; + } + + int paddingNeeded = alignment - charsWritten; + if (paddingNeeded > 0) + { + EnsureCapacityForAdditionalChars(paddingNeeded); + + if (leftAlign) + { + _chars.Slice(_pos, paddingNeeded).Fill(' '); + } + else + { + _chars.Slice(startingPos, charsWritten).CopyTo(_chars[(startingPos + paddingNeeded)..]); + _chars.Slice(startingPos, paddingNeeded).Fill(' '); + } + + _pos += paddingNeeded; + } + } + + /// Ensures has the capacity to store beyond . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EnsureCapacityForAdditionalChars(int additionalChars) + { + if (_chars.Length - _pos < additionalChars) + { + Grow(additionalChars); + } + } + + /// Fallback for fast path in when there's not enough space in the destination. + /// The string to write. + [MethodImpl(MethodImplOptions.NoInlining)] + private void GrowThenCopyString(string value) + { + Grow(value.Length); + value.CopyTo(_chars[_pos..]); + _pos += value.Length; + } + + /// Fallback for for when not enough space exists in the current buffer. + /// The span to write. + [MethodImpl(MethodImplOptions.NoInlining)] + private void GrowThenCopySpan(ReadOnlySpan value) + { + Grow(value.Length); + value.CopyTo(_chars[_pos..]); + _pos += value.Length; + } + + /// Grows to have the capacity to store at least beyond . + [MethodImpl(MethodImplOptions.NoInlining)] // keep consumers as streamlined as possible + private void Grow(int additionalChars) + { + // This method is called when the remaining space (_chars.Length - _pos) is + // insufficient to store a specific number of additional characters. Thus, we + // need to grow to at least that new total. GrowCore will handle growing by more + // than that if possible. + Debug.Assert(additionalChars > _chars.Length - _pos); + GrowCore((uint)_pos + (uint)additionalChars); + } + + /// Grows the size of . + [MethodImpl(MethodImplOptions.NoInlining)] // keep consumers as streamlined as possible + private void Grow() + { + // This method is called when the remaining space in _chars isn't sufficient to continue + // the operation. Thus, we need at least one character beyond _chars.Length. GrowCore + // will handle growing by more than that if possible. + GrowCore((uint)_chars.Length + 1); + } + + /// Grow the size of to at least the specified . + [MethodImpl(MethodImplOptions.AggressiveInlining)] // but reuse this grow logic directly in both of the above grow routines + private void GrowCore(uint requiredMinCapacity) + { + // We want the max of how much space we actually required and doubling our capacity (without going beyond the max allowed length). We + // also want to avoid asking for small arrays, to reduce the number of times we need to grow, and since we're working with unsigned + // ints that could technically overflow if someone tried to, for example, append a huge string to a huge string, we also clamp to int.MaxValue. + // Even if the array creation fails in such a case, we may later fail in ToStringAndClear. + + uint newCapacity = Math.Max(requiredMinCapacity, Math.Min((uint)_chars.Length * 2, 0x3FFFFFDF)); + int arraySize = (int)Math.Clamp(newCapacity, MinimumArrayPoolLength, int.MaxValue); + + char[] newArray = STArrayPool.Shared.Rent(arraySize); + _chars[.._pos].CopyTo(newArray); + + char[]? toReturn = _arrayToReturnToPool; + _chars = _arrayToReturnToPool = newArray; + + if (toReturn is not null) + { + STArrayPool.Shared.Return(toReturn); + } + } +} diff --git a/Projects/Server/Buffers/SpanReader.cs b/Projects/Server/Buffers/SpanReader.cs index d9459c8f2..cbff16597 100644 --- a/Projects/Server/Buffers/SpanReader.cs +++ b/Projects/Server/Buffers/SpanReader.cs @@ -166,7 +166,7 @@ namespace System.Buffers [MethodImpl(MethodImplOptions.AggressiveInlining)] public string ReadString(Encoding encoding, bool safeString = false, int fixedLength = -1) { - int sizeT = TextEncoding.GetByteLengthForEncoding(encoding); + int byteLength = encoding.GetByteLengthForEncoding(); bool isFixedLength = fixedLength > -1; @@ -174,7 +174,7 @@ namespace System.Buffers int size; if (isFixedLength) { - size = fixedLength * sizeT; + size = fixedLength * byteLength; if (size > Remaining) { throw new OutOfMemoryException(); @@ -183,8 +183,8 @@ namespace System.Buffers else { // In case the remaining is not evenly divisible - size = remaining - (remaining & (sizeT - 1)); - int index = _buffer.Slice(Position, size).IndexOfTerminator(sizeT); + size = remaining - (remaining & (byteLength - 1)); + int index = _buffer.Slice(Position, size).IndexOfTerminator(byteLength); size = index < 0 ? size : index; } diff --git a/Projects/Server/Buffers/SpanWriter.cs b/Projects/Server/Buffers/SpanWriter.cs index 89981cc5e..aaf7b17f3 100644 --- a/Projects/Server/Buffers/SpanWriter.cs +++ b/Projects/Server/Buffers/SpanWriter.cs @@ -16,6 +16,7 @@ using System.Buffers.Binary; using System.Data; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -275,21 +276,52 @@ public ref struct SpanWriter [MethodImpl(MethodImplOptions.AggressiveInlining)] public void WriteAscii(char chr) => Write((byte)chr); - public void WriteString(string value, Encoding encoding, int fixedLength = -1) where T : struct, IEquatable + public void WriteAscii( + ref RawInterpolatedStringHandler handler) { - int sizeT = Unsafe.SizeOf(); + Write(handler.Text, Encoding.ASCII); + handler.Clear(); + } - if (sizeT > 2) + public void WriteAscii( + IFormatProvider? formatProvider, + [InterpolatedStringHandlerArgument("formatProvider")] + ref RawInterpolatedStringHandler handler) + { + Write(handler.Text, Encoding.ASCII); + handler.Clear(); + } + + public void Write( + Encoding encoding, + ref RawInterpolatedStringHandler handler) + { + Write(handler.Text, encoding); + handler.Clear(); + } + + public void Write( + Encoding encoding, + IFormatProvider? formatProvider, + [InterpolatedStringHandlerArgument("formatProvider")] + ref RawInterpolatedStringHandler handler) + { + Write(handler.Text, encoding); + handler.Clear(); + } + + public void Write(ReadOnlySpan value, Encoding encoding, int fixedLength = -1) + { + var charLength = Math.Min(fixedLength > -1 ? fixedLength : value.Length, value.Length); + var src = value[..charLength]; + + var byteLength = encoding.GetByteLengthForEncoding(); + var byteCount = encoding.GetByteCount(src); + if (fixedLength > src.Length) { - throw new InvalidConstraintException("WriteString only accepts byte, sbyte, char, short, and ushort as a constraint"); + byteCount += (fixedLength - src.Length) * byteLength; } - value ??= string.Empty; - - var charLength = Math.Min(fixedLength > -1 ? fixedLength : value.Length, value.Length); - var src = value.AsSpan(0, charLength); - - var byteCount = fixedLength > -1 ? fixedLength * sizeT : encoding.GetByteCount(value); if (byteCount == 0) { return; @@ -302,7 +334,7 @@ public ref struct SpanWriter if (fixedLength > -1) { - var extra = fixedLength * sizeT - bytesWritten; + var extra = fixedLength * byteLength - bytesWritten; if (extra > 0) { Clear(extra); @@ -311,53 +343,53 @@ public ref struct SpanWriter } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void WriteLittleUni(string value) => WriteString(value, TextEncoding.UnicodeLE); + public void WriteLittleUni(string value) => Write(value, TextEncoding.UnicodeLE); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void WriteLittleUniNull(string value) { - WriteString(value, TextEncoding.UnicodeLE); + Write(value, TextEncoding.UnicodeLE); Write((ushort)0); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void WriteLittleUni(string value, int fixedLength) => WriteString(value, TextEncoding.UnicodeLE, fixedLength); + public void WriteLittleUni(string value, int fixedLength) => Write(value, TextEncoding.UnicodeLE, fixedLength); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void WriteBigUni(string value) => WriteString(value, TextEncoding.Unicode); + public void WriteBigUni(string value) => Write(value, TextEncoding.Unicode); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void WriteBigUniNull(string value) { - WriteString(value, TextEncoding.Unicode); + Write(value, TextEncoding.Unicode); Write((ushort)0); // '\0' } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void WriteBigUni(string value, int fixedLength) => WriteString(value, TextEncoding.Unicode, fixedLength); + public void WriteBigUni(string value, int fixedLength) => Write(value, TextEncoding.Unicode, fixedLength); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void WriteUTF8(string value) => WriteString(value, TextEncoding.UTF8); + public void WriteUTF8(string value) => Write(value, TextEncoding.UTF8); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void WriteUTF8Null(string value) { - WriteString(value, TextEncoding.UTF8); + Write(value, TextEncoding.UTF8); Write((byte)0); // '\0' } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void WriteAscii(string value) => WriteString(value, Encoding.ASCII); + public void WriteAscii(string value) => Write(value, Encoding.ASCII); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void WriteAsciiNull(string value) { - WriteString(value, Encoding.ASCII); + Write(value, Encoding.ASCII); Write((byte)0); // '\0' } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void WriteAscii(string value, int fixedLength) => WriteString(value, Encoding.ASCII, fixedLength); + public void WriteAscii(string value, int fixedLength) => Write(value, Encoding.ASCII, fixedLength); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Clear(int count) diff --git a/Projects/Server/Localization/LocalizationEntry.cs b/Projects/Server/Localization/LocalizationEntry.cs index 9bb2d4540..f91bcbce0 100644 --- a/Projects/Server/Localization/LocalizationEntry.cs +++ b/Projects/Server/Localization/LocalizationEntry.cs @@ -137,7 +137,7 @@ public class LocalizationEntry public LocalizationInterpolationHandler(int literalLength, int formattedCount, LocalizationEntry entry, out bool isValid) { _slices = entry.TextSlices; - _chars = _arrayToReturnToPool = ArrayPool.Shared.Rent(256); + _chars = _arrayToReturnToPool = STArrayPool.Shared.Rent(256); isValid = true; _pos = 0; @@ -159,7 +159,7 @@ public class LocalizationEntry if (Localization.TryGetLocalization(lang, number, out var entry)) { _slices = entry.TextSlices; - _chars = _arrayToReturnToPool = ArrayPool.Shared.Rent(256); + _chars = _arrayToReturnToPool = STArrayPool.Shared.Rent(256); isValid = true; } else @@ -509,10 +509,10 @@ public class LocalizationEntry [MethodImpl(MethodImplOptions.AggressiveInlining)] private void GrowCore(uint requiredMinCapacity) { - var newCapacity = Math.Max(requiredMinCapacity, Math.Min((uint)_chars.Length * 2, 1073741823)); + var newCapacity = Math.Max(requiredMinCapacity, Math.Min((uint)_chars.Length * 2, 0x3FFFFFDF)); var arraySize = (int)Math.Clamp(newCapacity, 256, int.MaxValue); - var newArray = ArrayPool.Shared.Rent(arraySize); + var newArray = STArrayPool.Shared.Rent(arraySize); _chars[.._pos].CopyTo(newArray); var toReturn = _arrayToReturnToPool; @@ -520,7 +520,7 @@ public class LocalizationEntry if (toReturn is not null) { - ArrayPool.Shared.Return(toReturn); + STArrayPool.Shared.Return(toReturn); } } @@ -533,7 +533,7 @@ public class LocalizationEntry this = default; // defensive clear if (toReturn is not null) { - ArrayPool.Shared.Return(toReturn); + STArrayPool.Shared.Return(toReturn); } } diff --git a/Projects/Server/Text/TextEncoding.cs b/Projects/Server/Text/TextEncoding.cs index c84e01def..41d6df2b2 100644 --- a/Projects/Server/Text/TextEncoding.cs +++ b/Projects/Server/Text/TextEncoding.cs @@ -105,7 +105,7 @@ namespace Server.Text public static int GetBytesUtf8(this ReadOnlySpan str, Span buffer) => UTF8.GetBytes(str, buffer); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetByteLengthForEncoding(Encoding encoding) => + public static int GetByteLengthForEncoding(this Encoding encoding) => encoding.BodyName switch { "utf-16BE" => 2,