## Summary - Adds proper Latin1 encoding support, replacing CP1252 usage throughout the codebase - Adds specialized, optimized string decoding methods with safe string filtering for each encoding type - Filters invalid Unicode characters (C0/C1 control codes, non-characters) by removal rather than replacement since the UO client renders nothing for these characters - Fixes UTF-16 null terminator position handling to correctly advance by 2 bytes ## Changes TextEncoding.cs - Added SearchValues-based invalid byte/char detection for efficient filtering - Added encoding-specific GetString methods: GetStringAscii, GetStringLatin1, GetStringUtf8, GetStringBigUni, GetStringLittleUni - Each method supports a safeString parameter for filtering invalid characters - Little-endian UTF-16 uses direct memory cast for zero-copy decoding on LE systems - Invalid characters are removed (not replaced with U+FFFD) since the client renders nothing for them SpanReader.cs - Added ReadLatin1() and ReadLatin1Safe() methods - Rewrote encoding-specific read methods to use optimized TextEncoding.GetString* methods - Fixed UTF-16 null terminator handling: position now correctly advances by byteLength (2) instead of 1 SpanWriter.cs - Added WriteLatin1 and WriteLatin1Null methods ## Packet Updates - Updated all packet code to use Latin1 encoding instead of CP1252 - Affected: account packets, equipment packets, menu packets, message packets, mobile packets, player packets, secure trade packets, vendor packets, gump packets, book packets, mahjong packets ## Filtering Behavior Invalid characters filtered in safe mode: ``` ┌───────────────┬────────────────────────┐ │ Range │ Description │ ├───────────────┼────────────────────────┤ │ 0x00-0x1F │ C0 control codes │ ├───────────────┼────────────────────────┤ │ 0x7F │ DEL │ ├───────────────┼────────────────────────┤ │ 0x80-0x9F │ C1 control codes │ ├───────────────┼────────────────────────┤ │ 0xFFFE-0xFFFF │ Unicode non-characters │ └───────────────┴────────────────────────┘ ``` Note: Surrogate pairs (0xD800-0xDFFF) are not filtered because proper validation requires context checking for paired vs unpaired surrogates. The UO client renders nothing for these anyway. ## Test Plan - All 631 Server.Tests pass - Verified client rendering behavior using TestUnicodeGump command (pages 1-5) - Confirmed U+FFFD, unpaired surrogates, and non-characters all render as blank in client - Verified Latin1 characters (0xA0-0xFF) display correctly - Verified C1 control codes (0x80-0x9F) are filtered and don't display
603 lines
15 KiB
C#
603 lines
15 KiB
C#
using System;
|
|
using System.Buffers;
|
|
using System.IO;
|
|
using Xunit;
|
|
|
|
namespace Server.Tests.Buffers;
|
|
|
|
public class SpanReaderTests
|
|
{
|
|
[Fact]
|
|
public void TestReadByte()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0x12, reader.ReadByte());
|
|
Assert.Equal(0x34, reader.ReadByte());
|
|
Assert.Equal(2, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadByteAtEnd()
|
|
{
|
|
Assert.Throws<EndOfStreamException>(
|
|
() =>
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12];
|
|
var reader = new SpanReader(buffer);
|
|
reader.ReadByte();
|
|
reader.ReadByte();
|
|
}
|
|
);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadBoolean()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0, 1, 2, 255];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.False(reader.ReadBoolean());
|
|
Assert.True(reader.ReadBoolean());
|
|
Assert.True(reader.ReadBoolean());
|
|
Assert.True(reader.ReadBoolean());
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadSByte()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0xFF, 0x7F];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(-1, reader.ReadSByte());
|
|
Assert.Equal(127, reader.ReadSByte());
|
|
Assert.Equal(2, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadInt16BigEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0x1234, reader.ReadInt16());
|
|
Assert.Equal(2, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadInt16LittleEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x34, 0x12];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0x1234, reader.ReadInt16LE());
|
|
Assert.Equal(2, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadInt16AtEnd()
|
|
{
|
|
Assert.Throws<EndOfStreamException>(
|
|
() =>
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12];
|
|
var reader = new SpanReader(buffer);
|
|
reader.ReadInt16();
|
|
}
|
|
);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadUInt16BigEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal((ushort)0x1234, reader.ReadUInt16());
|
|
Assert.Equal(2, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadUInt16LittleEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x34, 0x12];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal((ushort)0x1234, reader.ReadUInt16LE());
|
|
Assert.Equal(2, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadInt32BigEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34, 0x56, 0x78];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0x12345678, reader.ReadInt32());
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadInt32AtEnd()
|
|
{
|
|
Assert.Throws<EndOfStreamException>(
|
|
() =>
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34, 0x56];
|
|
var reader = new SpanReader(buffer);
|
|
reader.ReadInt32();
|
|
}
|
|
);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadUInt32BigEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34, 0x56, 0x78];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0x12345678u, reader.ReadUInt32());
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadUInt32LittleEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x78, 0x56, 0x34, 0x12];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0x12345678u, reader.ReadUInt32LE());
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadInt64BigEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0x123456789ABCDEF0L, reader.ReadInt64());
|
|
Assert.Equal(8, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadInt64AtEnd()
|
|
{
|
|
Assert.Throws<EndOfStreamException>(
|
|
() =>
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE];
|
|
var reader = new SpanReader(buffer);
|
|
reader.ReadInt64();
|
|
}
|
|
);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadUInt64BigEndian()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0x123456789ABCDEF0UL, reader.ReadUInt64());
|
|
Assert.Equal(8, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadAsciiString()
|
|
{
|
|
var buffer = "Hello"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadAscii();
|
|
|
|
Assert.Equal("Hello", result);
|
|
Assert.Equal(5, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadAsciiStringWithNull()
|
|
{
|
|
var buffer = "Hel\0o"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadAscii();
|
|
|
|
Assert.Equal("Hel", result);
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadAsciiStringFixedLength()
|
|
{
|
|
var buffer = "Hello\0\0\0\0\0"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadAscii(10);
|
|
|
|
Assert.Equal("Hello", result);
|
|
Assert.Equal(10, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadAsciiStringFixedLengthTooLarge()
|
|
{
|
|
Assert.Throws<EndOfStreamException>(
|
|
() =>
|
|
{
|
|
var buffer = "Hel"u8;
|
|
var reader = new SpanReader(buffer);
|
|
reader.ReadAscii(10);
|
|
}
|
|
);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadAsciiSafe()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [(byte)'H', (byte)'e', 0xFF, (byte)'l', (byte)'o'];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadAsciiSafe();
|
|
|
|
Assert.Equal("He?lo", result);
|
|
Assert.Equal(5, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadUTF8String()
|
|
{
|
|
var buffer = "Hello"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadUTF8();
|
|
|
|
Assert.Equal("Hello", result);
|
|
Assert.Equal(5, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadUTF8StringWithNull()
|
|
{
|
|
var buffer = "Hi\0Bye"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadUTF8();
|
|
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(3, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadLittleUniString()
|
|
{
|
|
var buffer = "H\0i\0"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadLittleUni();
|
|
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadLittleUniStringWithNull()
|
|
{
|
|
var buffer = "H\0i\0\0\0X\0"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadLittleUni();
|
|
|
|
Assert.Equal("Hi", result);
|
|
// Position 6: 4 bytes for "Hi" + 2 bytes for UTF-16 null terminator
|
|
Assert.Equal(6, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadLittleUniStringFixedLength()
|
|
{
|
|
var buffer = "H\0i\0\0\0\0\0"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadLittleUni(4);
|
|
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(8, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadLittleUniSafe()
|
|
{
|
|
// Test with C1 control code (0x85 = NEL) which should be filtered
|
|
ReadOnlySpan<byte> buffer = [(byte)'H', 0, 0x85, 0x00, (byte)'i', 0];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadLittleUniSafe();
|
|
|
|
// C1 control (0x0085) removed - client renders nothing for invalid chars
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(6, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadBigUniString()
|
|
{
|
|
var buffer = "\0H\0i"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadBigUni();
|
|
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadBigUniStringWithNull()
|
|
{
|
|
var buffer = "\0H\0i\0\0\0X"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadBigUni();
|
|
|
|
Assert.Equal("Hi", result);
|
|
// Position 6: 4 bytes for "Hi" + 2 bytes for UTF-16 null terminator
|
|
Assert.Equal(6, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadBigUniStringFixedLength()
|
|
{
|
|
var buffer = "\0H\0i\0\0\0\0"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadBigUni(4);
|
|
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(8, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadBigUniSafe()
|
|
{
|
|
// Test with C1 control code (0x0085 = NEL) which should be filtered
|
|
ReadOnlySpan<byte> buffer = [0, (byte)'H', 0x00, 0x85, 0, (byte)'i'];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadBigUniSafe();
|
|
|
|
// C1 control (0x0085) removed - client renders nothing for invalid chars
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(6, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestSeekBegin()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03, 0x04, 0x05];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
reader.ReadByte();
|
|
reader.ReadByte();
|
|
|
|
var pos = reader.Seek(0, SeekOrigin.Begin);
|
|
|
|
Assert.Equal(0, pos);
|
|
Assert.Equal(0, reader.Position);
|
|
Assert.Equal(0x01, reader.ReadByte());
|
|
}
|
|
|
|
[Fact]
|
|
public void TestSeekCurrent()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03, 0x04, 0x05];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
reader.ReadByte();
|
|
var pos = reader.Seek(2, SeekOrigin.Current);
|
|
|
|
Assert.Equal(3, pos);
|
|
Assert.Equal(3, reader.Position);
|
|
Assert.Equal(0x04, reader.ReadByte());
|
|
}
|
|
|
|
[Fact]
|
|
public void TestSeekEnd()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03, 0x04, 0x05];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var pos = reader.Seek(-2, SeekOrigin.End);
|
|
|
|
Assert.Equal(3, pos);
|
|
Assert.Equal(3, reader.Position);
|
|
Assert.Equal(0x04, reader.ReadByte());
|
|
}
|
|
|
|
[Fact]
|
|
public void TestSeekNegativeThrows()
|
|
{
|
|
Assert.Throws<ArgumentOutOfRangeException>(
|
|
() =>
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03];
|
|
var reader = new SpanReader(buffer);
|
|
reader.Seek(-1, SeekOrigin.Begin);
|
|
}
|
|
);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestSeekBeyondEndThrows()
|
|
{
|
|
Assert.Throws<ArgumentOutOfRangeException>(
|
|
() =>
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03];
|
|
var reader = new SpanReader(buffer);
|
|
reader.Seek(10, SeekOrigin.Begin);
|
|
}
|
|
);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestRead()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03, 0x04, 0x05];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Span<byte> dest = stackalloc byte[3];
|
|
var bytesRead = reader.Read(dest);
|
|
|
|
Assert.Equal(3, bytesRead);
|
|
Assert.Equal(3, reader.Position);
|
|
AssertThat.Equal(dest, [0x01, 0x02, 0x03]);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadPartial()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Span<byte> dest = stackalloc byte[5];
|
|
var bytesRead = reader.Read(dest);
|
|
|
|
Assert.Equal(3, bytesRead);
|
|
Assert.Equal(3, reader.Position);
|
|
AssertThat.Equal(dest[..3], [0x01, 0x02, 0x03]);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadEmpty()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Span<byte> dest = [];
|
|
var bytesRead = reader.Read(dest);
|
|
|
|
Assert.Equal(0, bytesRead);
|
|
Assert.Equal(0, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadAtEnd()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
reader.Seek(3, SeekOrigin.Begin);
|
|
|
|
Span<byte> dest = stackalloc byte[5];
|
|
var bytesRead = reader.Read(dest);
|
|
|
|
Assert.Equal(0, bytesRead);
|
|
Assert.Equal(3, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestLength()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03, 0x04, 0x05];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(5, reader.Length);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestPosition()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(0, reader.Position);
|
|
|
|
reader.ReadByte();
|
|
Assert.Equal(1, reader.Position);
|
|
|
|
reader.ReadUInt16();
|
|
Assert.Equal(3, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestRemaining()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03, 0x04, 0x05];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal(5, reader.Remaining);
|
|
|
|
reader.ReadByte();
|
|
Assert.Equal(4, reader.Remaining);
|
|
|
|
reader.ReadUInt16();
|
|
Assert.Equal(2, reader.Remaining);
|
|
|
|
reader.ReadUInt16();
|
|
Assert.Equal(0, reader.Remaining);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestBuffer()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var bufferProperty = reader.Buffer;
|
|
|
|
Assert.Equal(3, bufferProperty.Length);
|
|
AssertThat.Equal(bufferProperty, buffer);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadStringEmptyFixedLength()
|
|
{
|
|
ReadOnlySpan<byte> buffer = [0x01, 0x02, 0x03];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadAscii(0);
|
|
|
|
Assert.Equal("", result);
|
|
Assert.Equal(0, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadMultipleStringsWithNullTerminators()
|
|
{
|
|
var buffer = "AB\0CD\0EF"u8;
|
|
var reader = new SpanReader(buffer);
|
|
|
|
Assert.Equal("AB", reader.ReadAscii());
|
|
Assert.Equal("CD", reader.ReadAscii());
|
|
Assert.Equal("EF", reader.ReadAscii());
|
|
Assert.Equal(8, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadLittleUniOddByteCount()
|
|
{
|
|
// If buffer has odd number of bytes, the last byte should be ignored
|
|
ReadOnlySpan<byte> buffer = [(byte)'H', 0, (byte)'i', 0, 0xFF];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadLittleUni();
|
|
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
|
|
[Fact]
|
|
public void TestReadBigUniOddByteCount()
|
|
{
|
|
// If buffer has odd number of bytes, the last byte should be ignored
|
|
ReadOnlySpan<byte> buffer = [0, (byte)'H', 0, (byte)'i', 0xFF];
|
|
var reader = new SpanReader(buffer);
|
|
|
|
var result = reader.ReadBigUni();
|
|
|
|
Assert.Equal("Hi", result);
|
|
Assert.Equal(4, reader.Position);
|
|
}
|
|
}
|