feat: Adds cliloc support and fixes valuestringbuilder ctor (#1013)

- [X] Adds cliloc support using the following API:
```cs
public static class Localization
{
    string GetText(int number);
    string GetText(int number, string lang);
    string Format(int number, params object[] args);
    string Format(int number, string lang, params object[] args);

    string Format(int number, $"{arg1}{arg2}{arg3}");
    string Format(int number, lang, $"{arg1}{arg2}{arg3}");

    bool TryGetLocalization(int number, out LocalizationEntry entry);
    bool TryGetLocalization(int number, string lang, out LocalizationEntry entry);
}

public class LocalizationEntry
{
    string Language { get; }
    string Number { get; }
    string Text { get; }
    string?[] TextSlices { get; } // Used for string building
    string StringFormatter { get; }

    string Format(params object[] args);
    string Format($"{arg1}{arg2}{arg3}");
}
```

- [X] Fixes a bug with ValueStringBuilder and default initialization size
- [X] Optimizes ValueStringBuilder to use `ISpanFormattable`
- [X] Adds `Append<T>(T value);` support ValueStringBuilder
- [X] Adds `Append($"");` string interpolation support to ValueStringBuider
This commit is contained in:
Kamron Batman 2022-05-08 21:24:32 -07:00 committed by GitHub
parent 790cb5d733
commit be7da3a3dc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 1046 additions and 156 deletions

View file

@ -0,0 +1,201 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2022 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: Localization.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using Server.Buffers;
namespace Server;
public static class Localization
{
private const bool _loadLocalizationOnStartup = false;
public const string FallbackLanguage = "enu";
private static Dictionary<int, LocalizationEntry> _fallbackEntries;
private static Dictionary<string, Dictionary<int, LocalizationEntry>> _localizations = new();
public static void Configure()
{
if (_loadLocalizationOnStartup)
{
foreach (var file in Core.FindDataFileByPattern("cliloc.*"))
{
var fi = new FileInfo(file);
LoadClilocs(fi.Extension.ToLowerInvariant(), file);
}
}
}
public static Dictionary<int, LocalizationEntry> LoadClilocs(string lang) =>
LoadClilocs(lang, Core.FindDataFile($"cliloc.{lang}", false));
private static Dictionary<int, LocalizationEntry> LoadClilocs(string lang, string file)
{
Dictionary<int, LocalizationEntry> entries = _localizations[lang] = new Dictionary<int, LocalizationEntry>();
if (lang == FallbackLanguage)
{
_fallbackEntries = entries;
}
if (File.Exists(file))
{
using var fs = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read);
using var bin = new BinaryReader(fs);
bin.ReadInt32();
bin.ReadInt16();
byte[] buffer = null;
while (bin.BaseStream.Length != bin.BaseStream.Position)
{
var number = bin.ReadInt32();
var flag = bin.ReadByte(); // Original, Custom, Modified
var length = bin.ReadInt16();
if (buffer == null || buffer.Length < length)
{
buffer = GC.AllocateUninitializedArray<byte>(length);
}
var bytesRead = bin.Read(buffer, 0, length);
if (bytesRead != length)
{
throw new Exception($"Could not read enough bytes from {file}");
}
var text = Encoding.UTF8.GetString(buffer.AsSpan(0, length));
entries[number] = new LocalizationEntry(lang, number, text);
}
}
return entries;
}
/// <summary>
/// Returns the original text for a localization entry.
/// </summary>
/// <param name="number">Localization number</param>
/// <param name="lang">Language in ISO 6392 format</param>
/// <returns>Original text for the localizaton entry</returns>
public static string GetText(int number, string lang = FallbackLanguage) =>
TryGetLocalization(lang, number, out var entry) ? entry.Text : null;
public static string Format(int number, string lang = FallbackLanguage) => GetText(number, lang);
/// <summary>
/// Creates a formatted string of the localization entry using the <see cref="FallbackLanguage" />.
/// Uses <see cref="string.Format"/> under the hood.
/// Note: This method is not recommended since it uses almost double the memory and 50% more processing.
/// Instead use Format with string interpolation.
/// </summary>
/// <param name="number">Localization number</param>
/// <param name="args">An object array containing zero or more objects to format</param>
/// <returns>A copy of the localization text where the placeholder arguments have been replaced with string representations of the provided arguments</returns>
public static string Format(int number, params object[] args) =>
!TryGetLocalization(number, out var entry) ? null : string.Format(entry.StringFormatter, args);
/// <summary>
/// Creates a formatted string of the localization entry using the specified language.
/// Uses <see cref="string.Format"/> under the hood.
/// Note: This method is not recommended since it uses almost double the memory and 50% more processing.
/// Instead use Format with string interpolation.
/// </summary>
/// <param name="number">Localization number</param>
/// <param name="lang">Language in ISO 639-2 format</param>
/// <param name="args">An object array containing zero or more objects to format</param>
/// <returns>A copy of the localization text where the placeholder arguments have been replaced with string representations of the provided arguments</returns>
public static string Format(int number, string lang, params object[] args) =>
!TryGetLocalization(lang, number, out var entry) ? null : string.Format(entry.StringFormatter, args);
/// <summary>
/// Gets a localization entry using the <see cref="FallbackLanguage" />.
/// </summary>
/// <param name="number">Localization number</param>
/// <param name="entry">Localization entry retrieved</param>
/// <returns>True if the entry exists, otherwise false.</returns>
public static bool TryGetLocalization(int number, out LocalizationEntry entry) =>
TryGetLocalization(FallbackLanguage, number, out entry);
/// <summary>
/// Gets a localization entry.
/// </summary>
/// <param name="lang">Language in ISO 639-2 format</param>
/// <param name="number">Localization number</param>
/// <param name="entry">Localization entry retrieved</param>
/// <returns>True if the entry exists, otherwise false.</returns>
public static bool TryGetLocalization(string lang, int number, out LocalizationEntry entry)
{
if (lang != FallbackLanguage)
{
if (!_localizations.TryGetValue(lang, out var entries))
{
entries = LoadClilocs(lang);
}
if (entries.TryGetValue(number, out entry))
{
return true;
}
}
_fallbackEntries ??= LoadClilocs(FallbackLanguage);
return _fallbackEntries.TryGetValue(number, out entry);
}
/// <summary>
/// Creates a formatted string of the localization entry using the specified language.
/// Uses string interpolation under the hood. This method is preferably relative to the object array method signature.
/// Example:
/// Localization.Format(1073841, "jpn", $"{totalItems}{maxItems}{totalWeight}");
/// </summary>
/// <param name="lang">Language in ISO 639-2 format</param>
/// <param name="number">Localization number</param>
/// <param name="handler">interpolated string handler used by the compiler as a string builder during compilation</param>
/// <returns>A copy of the localization text where the placeholder arguments have been replaced with string representations of the provided interpolation arguments</returns>
public static PooledArraySpanFormattable Format(
int number, string lang,
[InterpolatedStringHandlerArgument("number", "lang")]
ref LocalizationInterpolationHandler handler
)
{
var chars = handler.ToPooledArray(out var length);
handler = default; // Defensive clear
return new PooledArraySpanFormattable(chars, length);
}
/// <summary>
/// Creates a formatted string of the localization entry using the <see cref="FallbackLanguage" />.
/// Uses string interpolation under the hood. This method is preferably relative to the object array method signature.
/// Example:
/// Localization.Format(1073841, $"{totalItems}{maxItems}{totalWeight}");
/// </summary>
/// <param name="number">Localization number</param>
/// <param name="handler">interpolated string handler used by the compiler as a string builder during compilation</param>
/// <returns>A copy of the localization text where the placeholder arguments have been replaced with string representations of the provided interpolation arguments</returns>
public static PooledArraySpanFormattable Format(
int number,
[InterpolatedStringHandlerArgument("number")]
ref LocalizationInterpolationHandler handler
)
{
var chars = handler.ToPooledArray(out var length);
handler = default; // Defensive clear
return new PooledArraySpanFormattable(chars, length);
}
}

View file

@ -0,0 +1,104 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2022 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: LocalizationEntry.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
using System.Runtime.CompilerServices;
using System.Text.RegularExpressions;
using Server.Buffers;
using Server.Collections;
namespace Server;
public class LocalizationEntry
{
private static readonly Regex _textRegex = new(
@"~(\d+)[_\w]+~",
RegexOptions.Compiled |
RegexOptions.IgnoreCase |
RegexOptions.Singleline |
RegexOptions.CultureInvariant
);
public string Language { get; }
public int Number { get; }
public string Text { get; }
public string[] TextSlices { get; }
public string StringFormatter { get; }
public LocalizationEntry(string lang, int number, string text)
{
Language = lang;
Number = number;
Text = text;
ParseText(text, out var textSlices, out var stringFormatter);
TextSlices = textSlices;
StringFormatter = stringFormatter;
}
private static void ParseText(string text, out string[] textSlices, out string stringFormatter)
{
bool hasMatch = false;
var prevIndex = 0;
var builder = new ValueStringBuilder(stackalloc char[256]);
using var queue = PooledRefQueue<string>.Create();
foreach (Match match in _textRegex.Matches(text))
{
if (prevIndex < match.Index)
{
var substr = text[prevIndex..match.Index];
builder.Append(substr);
queue.Enqueue(substr);
}
queue.Enqueue(null);
hasMatch = true;
builder.Append($"{{{int.Parse(match.Groups[1].Value) - 1}}}");
prevIndex = match.Index + match.Length;
}
if (prevIndex < text.Length - 1)
{
var substr = prevIndex == 0 ? text : text[prevIndex..];
builder.Append(substr);
queue.Enqueue(substr);
}
textSlices = queue.ToArray();
stringFormatter = hasMatch ? builder.ToString() : null;
builder.Dispose();
}
public string Format(params object[] args) => string.Format(StringFormatter, args);
/// <summary>
/// Creates a formatted string of the localization entry.
/// Uses string interpolation under the hood. This method is preferably relative to the object array method signature.
/// Example:
/// Format($"{totalItems}{maxItems}{totalWeight}");
/// </summary>
/// <param name="handler">interpolated string handler used by the compiler as a string builder during compilation</param>
/// <returns>A copy of the localization text where the placeholder arguments have been replaced with string representations of the provided interpolation arguments</returns>
public PooledArraySpanFormattable Format(
[InterpolatedStringHandlerArgument("")]
ref LocalizationInterpolationHandler handler
)
{
var chars = handler.ToPooledArray(out var length);
handler = default; // Defensive clear
return new PooledArraySpanFormattable(chars, length);
}
}

View file

@ -0,0 +1,429 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2022 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: LocalizationInterpolationHandler.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
#nullable enable
using System;
using System.Buffers;
using System.Runtime.CompilerServices;
namespace Server;
[InterpolatedStringHandler]
public ref struct LocalizationInterpolationHandler
{
private static string[] _empty = Array.Empty<string>();
private char[]? _arrayToReturnToPool;
private Span<char> _chars;
private int _pos;
private int _index;
private string?[] _slices;
private string? _current;
public LocalizationInterpolationHandler(int literalLength, int formattedCount, LocalizationEntry entry, out bool isValid)
{
_slices = entry.TextSlices;
_chars = _arrayToReturnToPool = ArrayPool<char>.Shared.Rent(256);
isValid = true;
_pos = 0;
_index = 0;
_current = null;
}
public LocalizationInterpolationHandler(int literalLength, int formattedCount, int number, out bool isValid)
: this(literalLength, formattedCount, number, Localization.FallbackLanguage, out isValid)
{
}
public LocalizationInterpolationHandler(int literalLength, int formattedCount, int number, string lang, out bool isValid)
{
if (Localization.TryGetLocalization(lang, number, out var entry))
{
_slices = entry.TextSlices;
_chars = _arrayToReturnToPool = ArrayPool<char>.Shared.Rent(256);
isValid = true;
}
else
{
_slices = _empty;
_chars = _arrayToReturnToPool = default;
isValid = false;
}
_pos = 0;
_index = 0;
_current = null;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool MoveNext()
{
if ((uint)_index >= (uint)_slices.Length)
{
return false;
}
_current = _slices[_index++];
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool ReadyToAppend()
{
if (!MoveNext())
{
return false;
}
if (_current == null)
{
return true;
}
AppendStringDirect(_current);
return MoveNext();
}
public void AppendLiteral(string value)
{
}
private void AppendStringDirect(string value)
{
if (value.TryCopyTo(_chars[_pos..]))
{
_pos += value.Length;
}
else
{
GrowThenCopyString(value);
}
}
public void AppendFormatted<T>(T value)
{
if (!ReadyToAppend())
{
return;
}
string? s;
if (value is IFormattable)
{
// If the value can format itself directly into our buffer, do so.
if (value is ISpanFormattable)
{
int charsWritten;
while (!((ISpanFormattable)value).TryFormat(_chars[_pos..], out charsWritten, default, default)) // constrained call avoiding boxing for value types
{
Grow();
}
_pos += charsWritten;
return;
}
s = ((IFormattable)value).ToString(format: null, default); // constrained call avoiding boxing for value types
}
else
{
s = value?.ToString();
}
if (s is not null)
{
AppendStringDirect(s);
}
}
public void AppendFormatted<T>(T value, string? format)
{
if (!ReadyToAppend())
{
return;
}
string? s;
if (value is IFormattable)
{
// If the value can format itself directly into our buffer, do so.
if (value is ISpanFormattable)
{
int charsWritten;
while (!((ISpanFormattable)value).TryFormat(_chars[_pos..], out charsWritten, format, default)) // constrained call avoiding boxing for value types
{
Grow();
}
_pos += charsWritten;
return;
}
s = ((IFormattable)value).ToString(format, default); // constrained call avoiding boxing for value types
}
else
{
s = value?.ToString();
}
if (s is not null)
{
AppendStringDirect(s);
}
}
public void AppendFormatted<T>(T value, int alignment)
{
if (!ReadyToAppend())
{
return;
}
var startingPos = _pos;
AppendFormatted(value);
if (alignment != 0)
{
AppendOrInsertAlignmentIfNeeded(startingPos, alignment);
}
}
public void AppendFormatted<T>(T value, int alignment, string? format)
{
if (!ReadyToAppend())
{
return;
}
var startingPos = _pos;
AppendFormatted(value, format);
if (alignment != 0)
{
AppendOrInsertAlignmentIfNeeded(startingPos, alignment);
}
}
public void AppendFormatted(ReadOnlySpan<char> value)
{
if (!ReadyToAppend())
{
return;
}
if (value.TryCopyTo(_chars[_pos..]))
{
_pos += value.Length;
}
else
{
GrowThenCopySpan(value);
}
}
public void AppendFormatted(ReadOnlySpan<char> value, int alignment, string? format = null)
{
if (!ReadyToAppend())
{
return;
}
var leftAlign = false;
if (alignment < 0)
{
leftAlign = true;
alignment = -alignment;
}
var paddingRequired = alignment - value.Length;
if (paddingRequired <= 0)
{
// The value is as large or larger than the required amount of padding,
// so just write the value.
AppendFormatted(value);
return;
}
// Write the value along with the appropriate padding.
EnsureCapacityForAdditionalChars(value.Length + paddingRequired);
if (leftAlign)
{
value.CopyTo(_chars[_pos..]);
_pos += value.Length;
_chars.Slice(_pos, paddingRequired).Fill(' ');
_pos += paddingRequired;
}
else
{
_chars.Slice(_pos, paddingRequired).Fill(' ');
_pos += paddingRequired;
value.CopyTo(_chars[_pos..]);
_pos += value.Length;
}
}
public void AppendFormatted(object? value, int alignment = 0, string? format = null) =>
AppendFormatted<object?>(value, alignment, format);
public void AppendFormatted(string? value)
{
if (ReadyToAppend())
{
if (value?.TryCopyTo(_chars[_pos..]) == true)
{
_pos += value.Length;
}
else
{
AppendFormattedSlow(value);
}
}
}
public void AppendFormatted(string? value, int alignment, string? format = null) =>
AppendFormatted<string?>(value, alignment, format);
private void AppendOrInsertAlignmentIfNeeded(int startingPos, int alignment)
{
var charsWritten = _pos - startingPos;
var leftAlign = false;
if (alignment < 0)
{
leftAlign = true;
alignment = -alignment;
}
var paddingNeeded = alignment - charsWritten;
if (paddingNeeded > 0)
{
EnsureCapacityForAdditionalChars(paddingNeeded);
if (leftAlign)
{
_chars.Slice(_pos, paddingNeeded).Fill(' ');
}
else
{
_chars.Slice(startingPos, charsWritten).CopyTo(_chars[(startingPos + paddingNeeded)..]);
_chars.Slice(startingPos, paddingNeeded).Fill(' ');
}
_pos += paddingNeeded;
}
}
[MethodImpl(MethodImplOptions.NoInlining)]
private void AppendFormattedSlow(string? value)
{
if (value is not null)
{
EnsureCapacityForAdditionalChars(value.Length);
value.CopyTo(_chars[_pos..]);
_pos += value.Length;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void EnsureCapacityForAdditionalChars(int additionalChars)
{
if (_chars.Length - _pos < additionalChars)
{
Grow(additionalChars);
}
}
[MethodImpl(MethodImplOptions.NoInlining)]
private void GrowThenCopyString(string value)
{
Grow(value.Length);
value.CopyTo(_chars[_pos..]);
_pos += value.Length;
}
[MethodImpl(MethodImplOptions.NoInlining)]
private void GrowThenCopySpan(ReadOnlySpan<char> value)
{
Grow(value.Length);
value.CopyTo(_chars[_pos..]);
_pos += value.Length;
}
[MethodImpl(MethodImplOptions.NoInlining)] // keep consumers as streamlined as possible
private void Grow(int additionalChars)
{
GrowCore((uint)_pos + (uint)additionalChars);
}
[MethodImpl(MethodImplOptions.NoInlining)] // keep consumers as streamlined as possible
private void Grow()
{
GrowCore((uint)_chars.Length + 1);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void GrowCore(uint requiredMinCapacity)
{
var newCapacity = Math.Max(requiredMinCapacity, Math.Min((uint)_chars.Length * 2, 1073741823));
var arraySize = (int)Math.Clamp(newCapacity, 256, int.MaxValue);
var newArray = ArrayPool<char>.Shared.Rent(arraySize);
_chars[.._pos].CopyTo(newArray);
var toReturn = _arrayToReturnToPool;
_chars = _arrayToReturnToPool = newArray;
if (toReturn is not null)
{
ArrayPool<char>.Shared.Return(toReturn);
}
}
internal ReadOnlySpan<char> Text => _chars[.._pos];
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal void Clear()
{
var toReturn = _arrayToReturnToPool;
this = default; // defensive clear
if (toReturn is not null)
{
ArrayPool<char>.Shared.Return(toReturn);
}
}
public string ToStringAndClear()
{
if (MoveNext() && _current != null)
{
AppendStringDirect(_current);
}
var result = new string(Text);
Clear();
return result;
}
public char[] ToPooledArray(out int length)
{
if (MoveNext() && _current != null)
{
AppendStringDirect(_current);
}
length = _pos;
return _arrayToReturnToPool;
}
}