feat: Adds Latin1 text support (#2317)
## Summary - Adds proper Latin1 encoding support, replacing CP1252 usage throughout the codebase - Adds specialized, optimized string decoding methods with safe string filtering for each encoding type - Filters invalid Unicode characters (C0/C1 control codes, non-characters) by removal rather than replacement since the UO client renders nothing for these characters - Fixes UTF-16 null terminator position handling to correctly advance by 2 bytes ## Changes TextEncoding.cs - Added SearchValues-based invalid byte/char detection for efficient filtering - Added encoding-specific GetString methods: GetStringAscii, GetStringLatin1, GetStringUtf8, GetStringBigUni, GetStringLittleUni - Each method supports a safeString parameter for filtering invalid characters - Little-endian UTF-16 uses direct memory cast for zero-copy decoding on LE systems - Invalid characters are removed (not replaced with U+FFFD) since the client renders nothing for them SpanReader.cs - Added ReadLatin1() and ReadLatin1Safe() methods - Rewrote encoding-specific read methods to use optimized TextEncoding.GetString* methods - Fixed UTF-16 null terminator handling: position now correctly advances by byteLength (2) instead of 1 SpanWriter.cs - Added WriteLatin1 and WriteLatin1Null methods ## Packet Updates - Updated all packet code to use Latin1 encoding instead of CP1252 - Affected: account packets, equipment packets, menu packets, message packets, mobile packets, player packets, secure trade packets, vendor packets, gump packets, book packets, mahjong packets ## Filtering Behavior Invalid characters filtered in safe mode: ``` ┌───────────────┬────────────────────────┐ │ Range │ Description │ ├───────────────┼────────────────────────┤ │ 0x00-0x1F │ C0 control codes │ ├───────────────┼────────────────────────┤ │ 0x7F │ DEL │ ├───────────────┼────────────────────────┤ │ 0x80-0x9F │ C1 control codes │ ├───────────────┼────────────────────────┤ │ 0xFFFE-0xFFFF │ Unicode non-characters │ └───────────────┴────────────────────────┘ ``` Note: Surrogate pairs (0xD800-0xDFFF) are not filtered because proper validation requires context checking for paired vs unpaired surrogates. The UO client renders nothing for these anyway. ## Test Plan - All 631 Server.Tests pass - Verified client rendering behavior using TestUnicodeGump command (pages 1-5) - Confirmed U+FFFD, unpaired surrogates, and non-characters all render as blank in client - Verified Latin1 characters (0xA0-0xFF) display correctly - Verified C1 control codes (0x80-0x9F) are filtered and don't display
This commit is contained in:
Kamron Batman
GitHub
parent
1e4c32c809
commit
0404251638
27 changed files with 735 additions and 96 deletions
|
|
@ -247,15 +247,10 @@ public static class AccountHandler
|
|||
username.Length > 0 &&
|
||||
// Usernames must not start with a space, end with a space, or end with a period
|
||||
!username.StartsWith(' ') && !username.EndsWith(' ') && !username.EndsWith('.') &&
|
||||
// Usernames must only contain characters [0x20 -> 0x7E], and not contain any forbidden characters
|
||||
!username.ContainsAnyExceptInRange((char)0x20, (char)0x7E) &&
|
||||
!username.ContainsAny(ForbiddenChars);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsValidPassword(ReadOnlySpan<char> password) =>
|
||||
password.Length > 0 &&
|
||||
// Passwords must have characters [0x20 -> 0x7E]
|
||||
!password.ContainsAnyExceptInRange((char)0x20, (char)0x7E);
|
||||
public static bool IsValidPassword(ReadOnlySpan<char> password) => password.Length > 0;
|
||||
|
||||
private static Account CreateAccount(NetState state, string username, string password)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -577,7 +577,7 @@ public ref struct GumpLayoutBuilder
|
|||
{
|
||||
if (!value.IsEmpty)
|
||||
{
|
||||
_bytesWritten += value.GetBytesAscii(_layoutBuffer.AsSpan(_bytesWritten));
|
||||
_bytesWritten += value.GetBytesLatin1(_layoutBuffer.AsSpan(_bytesWritten));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ public class GumpHtmlLocalized : GumpEntry
|
|||
}
|
||||
case GumpHtmlLocalizedType.Args:
|
||||
{
|
||||
writer.WriteAscii(
|
||||
writer.WriteLatin1(
|
||||
$"{{ xmfhtmltok {X} {Y} {Width} {Height} {(Background ? "1" : "0")} {(Scrollbar ? "1" : "0")} {Color} {Number} @{Args}@ }}"
|
||||
);
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -34,6 +34,13 @@ public class GumpTooltip : GumpEntry
|
|||
|
||||
public override void AppendTo(ref SpanWriter writer, OrderedSet<string> strings, ref int entries, ref int switches)
|
||||
{
|
||||
writer.WriteAscii(string.IsNullOrEmpty(Args) ? $"{{ tooltip {Number} }}" : $"{{ tooltip {Number} @{Args}@ }}");
|
||||
if (string.IsNullOrEmpty(Args))
|
||||
{
|
||||
writer.WriteAscii($"{{ tooltip {Number} }}");
|
||||
}
|
||||
else
|
||||
{
|
||||
writer.WriteLatin1($"{{ tooltip {Number} @{Args}@ }}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,9 +73,9 @@ public static class OutgoingGumpPackets
|
|||
writer.Write(serial);
|
||||
writer.Write((short)gumpId);
|
||||
writer.Write((short)(unknown.Length + 1));
|
||||
writer.WriteAsciiNull(unknown);
|
||||
writer.WriteLatin1Null(unknown);
|
||||
writer.Write((short)(caption.Length + 1));
|
||||
writer.WriteAsciiNull(caption);
|
||||
writer.WriteLatin1Null(caption);
|
||||
|
||||
ns.Send(writer.Span);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*************************************************************************
|
||||
* ModernUO *
|
||||
* Copyright 2019-2024 - ModernUO Development Team *
|
||||
* Copyright 2019-2026 - ModernUO Development Team *
|
||||
* Email: hi@modernuo.com *
|
||||
* File: RelayInfo.cs *
|
||||
* *
|
||||
|
|
@ -48,8 +48,6 @@ public readonly ref struct RelayInfo
|
|||
public string GetTextEntry(int entryId)
|
||||
{
|
||||
var index = _textIds.IndexOf((ushort)entryId);
|
||||
return index == -1
|
||||
? default
|
||||
: TextEncoding.GetString(_textBlock[_textRanges[index]], TextEncoding.Unicode, true);
|
||||
return index == -1 ? null : TextEncoding.GetStringBigUni(_textBlock[_textRanges[index]], true);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,8 +41,8 @@ namespace Server.Items
|
|||
|
||||
reader.Seek(4, SeekOrigin.Current); // Skip flags and page count
|
||||
|
||||
var title = reader.ReadAsciiSafe(60);
|
||||
var author = reader.ReadAsciiSafe(30);
|
||||
var title = reader.ReadLatin1Safe(60);
|
||||
var author = reader.ReadLatin1Safe(30);
|
||||
|
||||
book.Title = title.FixHtml();
|
||||
book.Author = author.FixHtml();
|
||||
|
|
|
|||
|
|
@ -346,7 +346,7 @@ namespace Server.Engines.Mahjong
|
|||
writer.Write((byte)0);
|
||||
|
||||
writer.Write(players.IsPublic(i));
|
||||
writer.WriteAscii(m?.Name ?? "", 30);
|
||||
writer.WriteLatin1(m?.Name ?? "", 30);
|
||||
|
||||
writer.Write(m == null || !players.IsInGamePlayer(i));
|
||||
count++;
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ public static class IncomingAccountPackets
|
|||
var unk2 = reader.ReadInt32();
|
||||
int unk3 = reader.ReadByte();
|
||||
*/
|
||||
var name = reader.ReadAscii(30);
|
||||
var name = reader.ReadLatin1Safe(30);
|
||||
|
||||
reader.Seek(2, SeekOrigin.Current);
|
||||
var flags = reader.ReadInt32();
|
||||
|
|
@ -371,8 +371,8 @@ public static class IncomingAccountPackets
|
|||
state.Version = ap.Version;
|
||||
state.Seeded = true;
|
||||
|
||||
var username = reader.ReadAscii(30);
|
||||
var password = reader.ReadAscii(30);
|
||||
var username = reader.ReadLatin1Safe(30);
|
||||
var password = reader.ReadLatin1Safe(30);
|
||||
|
||||
var e = new GameServer.GameLoginEventArgs(state, username, password);
|
||||
|
||||
|
|
@ -446,8 +446,8 @@ public static class IncomingAccountPackets
|
|||
|
||||
state.SentFirstPacket = true;
|
||||
|
||||
var username = reader.ReadAscii(30);
|
||||
var password = reader.ReadAscii(30);
|
||||
var username = reader.ReadLatin1Safe(30);
|
||||
var password = reader.ReadLatin1Safe(30);
|
||||
|
||||
var accountLoginEventArgs = new AccountLoginEventArgs(state, username, password);
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ public static class IncomingMessagePackets
|
|||
var type = (MessageType)reader.ReadByte();
|
||||
int hue = reader.ReadInt16();
|
||||
reader.ReadInt16(); // font
|
||||
var text = reader.ReadAsciiSafe().Trim();
|
||||
var text = reader.ReadLatin1Safe().Trim();
|
||||
|
||||
if (text.Length is <= 0 or > 128)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ public static class IncomingMobilePackets
|
|||
|
||||
if (targ != null)
|
||||
{
|
||||
RenameRequests.RenameRequest(from, targ, reader.ReadAsciiSafe());
|
||||
RenameRequests.RenameRequest(from, targ, reader.ReadLatin1Safe());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -223,7 +223,7 @@ public static class IncomingPlayerPackets
|
|||
var serial = reader.ReadUInt32();
|
||||
var prompt = reader.ReadInt32();
|
||||
var type = reader.ReadInt32();
|
||||
var text = reader.ReadAsciiSafe();
|
||||
var text = reader.ReadLatin1Safe();
|
||||
|
||||
if (text.Length > 128)
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue