feat: Adds Latin1 text support (#2317)
## Summary - Adds proper Latin1 encoding support, replacing CP1252 usage throughout the codebase - Adds specialized, optimized string decoding methods with safe string filtering for each encoding type - Filters invalid Unicode characters (C0/C1 control codes, non-characters) by removal rather than replacement since the UO client renders nothing for these characters - Fixes UTF-16 null terminator position handling to correctly advance by 2 bytes ## Changes TextEncoding.cs - Added SearchValues-based invalid byte/char detection for efficient filtering - Added encoding-specific GetString methods: GetStringAscii, GetStringLatin1, GetStringUtf8, GetStringBigUni, GetStringLittleUni - Each method supports a safeString parameter for filtering invalid characters - Little-endian UTF-16 uses direct memory cast for zero-copy decoding on LE systems - Invalid characters are removed (not replaced with U+FFFD) since the client renders nothing for them SpanReader.cs - Added ReadLatin1() and ReadLatin1Safe() methods - Rewrote encoding-specific read methods to use optimized TextEncoding.GetString* methods - Fixed UTF-16 null terminator handling: position now correctly advances by byteLength (2) instead of 1 SpanWriter.cs - Added WriteLatin1 and WriteLatin1Null methods ## Packet Updates - Updated all packet code to use Latin1 encoding instead of CP1252 - Affected: account packets, equipment packets, menu packets, message packets, mobile packets, player packets, secure trade packets, vendor packets, gump packets, book packets, mahjong packets ## Filtering Behavior Invalid characters filtered in safe mode: ``` ┌───────────────┬────────────────────────┐ │ Range │ Description │ ├───────────────┼────────────────────────┤ │ 0x00-0x1F │ C0 control codes │ ├───────────────┼────────────────────────┤ │ 0x7F │ DEL │ ├───────────────┼────────────────────────┤ │ 0x80-0x9F │ C1 control codes │ ├───────────────┼────────────────────────┤ │ 0xFFFE-0xFFFF │ Unicode non-characters │ └───────────────┴────────────────────────┘ ``` Note: Surrogate pairs (0xD800-0xDFFF) are not filtered because proper validation requires context checking for paired vs unpaired surrogates. The UO client renders nothing for these anyway. ## Test Plan - All 631 Server.Tests pass - Verified client rendering behavior using TestUnicodeGump command (pages 1-5) - Confirmed U+FFFD, unpaired surrogates, and non-characters all render as blank in client - Verified Latin1 characters (0xA0-0xFF) display correctly - Verified C1 control codes (0x80-0x9F) are filtered and don't display
This commit is contained in:
Kamron Batman
GitHub
parent
1e4c32c809
commit
0404251638
27 changed files with 735 additions and 96 deletions
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2025 - ModernUO Development Team *
|
||||
* Copyright 2019-2026 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: SpanReader.cs *
|
||||
* *
|
||||
|
|
@ -207,49 +207,241 @@ public ref struct SpanReader
|
|||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadLittleUniSafe(int fixedLength) => ReadString(TextEncoding.UnicodeLE, true, fixedLength);
|
||||
public string ReadLittleUniSafe(int fixedLength) => ReadStringLittleUni(true, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadLittleUniSafe() => ReadString(TextEncoding.UnicodeLE, true);
|
||||
public string ReadLittleUniSafe() => ReadStringLittleUni(true);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadLittleUni(int fixedLength) => ReadString(TextEncoding.UnicodeLE, false, fixedLength);
|
||||
public string ReadLittleUni(int fixedLength) => ReadStringLittleUni(false, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadLittleUni() => ReadString(TextEncoding.UnicodeLE);
|
||||
public string ReadLittleUni() => ReadStringLittleUni(false);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadBigUniSafe(int fixedLength) => ReadString(TextEncoding.Unicode, true, fixedLength);
|
||||
public string ReadBigUniSafe(int fixedLength) => ReadStringBigUni(true, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadBigUniSafe() => ReadString(TextEncoding.Unicode, true);
|
||||
public string ReadBigUniSafe() => ReadStringBigUni(true);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadBigUni(int fixedLength) => ReadString(TextEncoding.Unicode, false, fixedLength);
|
||||
public string ReadBigUni(int fixedLength) => ReadStringBigUni(false, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadBigUni() => ReadString(TextEncoding.Unicode);
|
||||
public string ReadBigUni() => ReadStringBigUni(false);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadUTF8Safe(int fixedLength) => ReadString(TextEncoding.UTF8, true, fixedLength);
|
||||
public string ReadUTF8Safe(int fixedLength) => ReadStringUtf8(true, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadUTF8Safe() => ReadString(TextEncoding.UTF8, true);
|
||||
public string ReadUTF8Safe() => ReadStringUtf8(true);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadUTF8() => ReadString(TextEncoding.UTF8);
|
||||
public string ReadUTF8(int fixedLength) => ReadStringUtf8(false, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadAsciiSafe(int fixedLength) => ReadString(Encoding.ASCII, true, fixedLength);
|
||||
public string ReadUTF8() => ReadStringUtf8(false);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadAsciiSafe() => ReadString(Encoding.ASCII, true);
|
||||
public string ReadAsciiSafe(int fixedLength) => ReadStringAscii(true, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadAscii(int fixedLength) => ReadString(Encoding.ASCII, false, fixedLength);
|
||||
public string ReadAsciiSafe() => ReadStringAscii(true);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadAscii() => ReadString(Encoding.ASCII);
|
||||
public string ReadAscii(int fixedLength) => ReadStringAscii(false, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadAscii() => ReadStringAscii(false);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadLatin1Safe(int fixedLength) => ReadStringLatin1(true, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadLatin1Safe() => ReadStringLatin1(true);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadLatin1(int fixedLength) => ReadStringLatin1(false, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public string ReadLatin1() => ReadStringLatin1(false);
|
||||
|
||||
/// <summary>
|
||||
/// Reads a big-endian UTF-16 string, filtering control codes and non-characters in safe mode.
|
||||
/// </summary>
|
||||
private string ReadStringBigUni(bool safeString, int fixedLength = -1)
|
||||
{
|
||||
if (fixedLength == 0)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
const int byteLength = 2;
|
||||
var isFixedLength = fixedLength > -1;
|
||||
|
||||
var remaining = Remaining;
|
||||
int size;
|
||||
if (isFixedLength)
|
||||
{
|
||||
size = fixedLength * byteLength;
|
||||
if (size > remaining)
|
||||
{
|
||||
throw new EndOfStreamException("Cannot read past the end of the buffer.");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Ensure even number of bytes
|
||||
size = remaining - (remaining & 1);
|
||||
}
|
||||
|
||||
var span = _buffer.Slice(Position, size);
|
||||
var index = span.IndexOfTerminator(byteLength);
|
||||
|
||||
if (index > -1)
|
||||
{
|
||||
span = _buffer.Slice(Position, index);
|
||||
}
|
||||
|
||||
Position += isFixedLength || index < 0 ? size : index + byteLength;
|
||||
|
||||
return TextEncoding.GetStringBigUni(span, safeString);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a little-endian UTF-16 string, filtering control codes and non-characters in safe mode.
|
||||
/// </summary>
|
||||
private string ReadStringLittleUni(bool safeString, int fixedLength = -1)
|
||||
{
|
||||
if (fixedLength == 0)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
const int byteLength = 2;
|
||||
var isFixedLength = fixedLength > -1;
|
||||
|
||||
var remaining = Remaining;
|
||||
int size;
|
||||
if (isFixedLength)
|
||||
{
|
||||
size = fixedLength * byteLength;
|
||||
if (size > remaining)
|
||||
{
|
||||
throw new EndOfStreamException("Cannot read past the end of the buffer.");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Ensure even number of bytes
|
||||
size = remaining - (remaining & 1);
|
||||
}
|
||||
|
||||
var span = _buffer.Slice(Position, size);
|
||||
var index = span.IndexOfTerminator(byteLength);
|
||||
|
||||
if (index > -1)
|
||||
{
|
||||
span = _buffer.Slice(Position, index);
|
||||
}
|
||||
|
||||
Position += isFixedLength || index < 0 ? size : index + byteLength;
|
||||
|
||||
return TextEncoding.GetStringLittleUni(span, safeString);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads an ASCII string, filtering C0 control codes and DEL in safe mode.
|
||||
/// </summary>
|
||||
private string ReadStringAscii(bool safeString, int fixedLength = -1)
|
||||
{
|
||||
if (fixedLength == 0)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var isFixedLength = fixedLength > -1;
|
||||
|
||||
int size = isFixedLength ? fixedLength : Remaining;
|
||||
if (size > Remaining)
|
||||
{
|
||||
throw new EndOfStreamException("Cannot read past the end of the buffer.");
|
||||
}
|
||||
|
||||
var span = _buffer.Slice(Position, size);
|
||||
var index = span.IndexOf((byte)0);
|
||||
|
||||
if (index > -1)
|
||||
{
|
||||
span = _buffer.Slice(Position, index);
|
||||
}
|
||||
|
||||
Position += isFixedLength || index < 0 ? size : index + 1;
|
||||
|
||||
return TextEncoding.GetStringAscii(span, safeString);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a UTF-8 string, filtering control codes and non-characters in safe mode.
|
||||
/// </summary>
|
||||
private string ReadStringUtf8(bool safeString, int fixedLength = -1)
|
||||
{
|
||||
if (fixedLength == 0)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var isFixedLength = fixedLength > -1;
|
||||
|
||||
int size = isFixedLength ? fixedLength : Remaining;
|
||||
if (size > Remaining)
|
||||
{
|
||||
throw new EndOfStreamException("Cannot read past the end of the buffer.");
|
||||
}
|
||||
|
||||
var span = _buffer.Slice(Position, size);
|
||||
var index = span.IndexOf((byte)0);
|
||||
|
||||
if (index > -1)
|
||||
{
|
||||
span = _buffer.Slice(Position, index);
|
||||
}
|
||||
|
||||
Position += isFixedLength || index < 0 ? size : index + 1;
|
||||
|
||||
return TextEncoding.GetStringUtf8(span, safeString);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a Latin1 string, filtering C0/C1 control codes in safe mode.
|
||||
/// </summary>
|
||||
private string ReadStringLatin1(bool safeString, int fixedLength = -1)
|
||||
{
|
||||
if (fixedLength == 0)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var isFixedLength = fixedLength > -1;
|
||||
|
||||
int size = isFixedLength ? fixedLength : Remaining;
|
||||
if (size > Remaining)
|
||||
{
|
||||
throw new EndOfStreamException("Cannot read past the end of the buffer.");
|
||||
}
|
||||
|
||||
var span = _buffer.Slice(Position, size);
|
||||
var index = span.IndexOf((byte)0);
|
||||
|
||||
if (index > -1)
|
||||
{
|
||||
span = _buffer.Slice(Position, index);
|
||||
}
|
||||
|
||||
Position += isFixedLength || index < 0 ? size : index + 1;
|
||||
|
||||
return TextEncoding.GetStringLatin1(span, safeString);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int Seek(int offset, SeekOrigin origin)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2025 - ModernUO Development Team *
|
||||
* Copyright 2019-2026 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: SpanWriter.cs *
|
||||
* *
|
||||
|
|
@ -273,8 +273,7 @@ public ref struct SpanWriter
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void WriteAscii(char chr) => Write((byte)chr);
|
||||
|
||||
public void WriteAscii(
|
||||
ref RawInterpolatedStringHandler handler)
|
||||
public void WriteAscii(ref RawInterpolatedStringHandler handler)
|
||||
{
|
||||
Write(handler.Text, Encoding.ASCII);
|
||||
handler.Clear();
|
||||
|
|
@ -289,9 +288,22 @@ public ref struct SpanWriter
|
|||
handler.Clear();
|
||||
}
|
||||
|
||||
public void Write(
|
||||
Encoding encoding,
|
||||
public void WriteLatin1(ref RawInterpolatedStringHandler handler)
|
||||
{
|
||||
Write(handler.Text, Encoding.Latin1);
|
||||
handler.Clear();
|
||||
}
|
||||
|
||||
public void WriteLatin1(
|
||||
IFormatProvider? formatProvider,
|
||||
[InterpolatedStringHandlerArgument("formatProvider")]
|
||||
ref RawInterpolatedStringHandler handler)
|
||||
{
|
||||
Write(handler.Text, Encoding.Latin1);
|
||||
handler.Clear();
|
||||
}
|
||||
|
||||
public void Write(Encoding encoding, ref RawInterpolatedStringHandler handler)
|
||||
{
|
||||
Write(handler.Text, encoding);
|
||||
handler.Clear();
|
||||
|
|
@ -388,6 +400,19 @@ public ref struct SpanWriter
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void WriteAscii(string value, int fixedLength) => Write(value, Encoding.ASCII, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void WriteLatin1(string value) => Write(value, Encoding.Latin1);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void WriteLatin1(string value, int fixedLength) => Write(value, Encoding.Latin1, fixedLength);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void WriteLatin1Null(string value)
|
||||
{
|
||||
Write(value, Encoding.Latin1);
|
||||
Write((byte)0); // '\0'
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Clear(int count)
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue