feat: Adds Latin1 text support (#2317)
## Summary - Adds proper Latin1 encoding support, replacing CP1252 usage throughout the codebase - Adds specialized, optimized string decoding methods with safe string filtering for each encoding type - Filters invalid Unicode characters (C0/C1 control codes, non-characters) by removal rather than replacement since the UO client renders nothing for these characters - Fixes UTF-16 null terminator position handling to correctly advance by 2 bytes ## Changes TextEncoding.cs - Added SearchValues-based invalid byte/char detection for efficient filtering - Added encoding-specific GetString methods: GetStringAscii, GetStringLatin1, GetStringUtf8, GetStringBigUni, GetStringLittleUni - Each method supports a safeString parameter for filtering invalid characters - Little-endian UTF-16 uses direct memory cast for zero-copy decoding on LE systems - Invalid characters are removed (not replaced with U+FFFD) since the client renders nothing for them SpanReader.cs - Added ReadLatin1() and ReadLatin1Safe() methods - Rewrote encoding-specific read methods to use optimized TextEncoding.GetString* methods - Fixed UTF-16 null terminator handling: position now correctly advances by byteLength (2) instead of 1 SpanWriter.cs - Added WriteLatin1 and WriteLatin1Null methods ## Packet Updates - Updated all packet code to use Latin1 encoding instead of CP1252 - Affected: account packets, equipment packets, menu packets, message packets, mobile packets, player packets, secure trade packets, vendor packets, gump packets, book packets, mahjong packets ## Filtering Behavior Invalid characters filtered in safe mode: ``` ┌───────────────┬────────────────────────┐ │ Range │ Description │ ├───────────────┼────────────────────────┤ │ 0x00-0x1F │ C0 control codes │ ├───────────────┼────────────────────────┤ │ 0x7F │ DEL │ ├───────────────┼────────────────────────┤ │ 0x80-0x9F │ C1 control codes │ ├───────────────┼────────────────────────┤ │ 0xFFFE-0xFFFF │ Unicode non-characters │ └───────────────┴────────────────────────┘ ``` Note: Surrogate pairs (0xD800-0xDFFF) are not filtered because proper validation requires context checking for paired vs unpaired surrogates. The UO client renders nothing for these anyway. ## Test Plan - All 631 Server.Tests pass - Verified client rendering behavior using TestUnicodeGump command (pages 1-5) - Confirmed U+FFFD, unpaired surrogates, and non-characters all render as blank in client - Verified Latin1 characters (0xA0-0xFF) display correctly - Verified C1 control codes (0x80-0x9F) are filtered and don't display
This commit is contained in:
Kamron Batman
GitHub
parent
1e4c32c809
commit
0404251638
27 changed files with 735 additions and 96 deletions
|
|
@ -89,7 +89,7 @@ public static class OutgoingAccountPackets
|
|||
var name = (m.RawName?.Trim()).DefaultIfNullOrEmpty("-no name-");
|
||||
|
||||
count++;
|
||||
writer.WriteAscii(name, 30);
|
||||
writer.WriteLatin1(name, 30);
|
||||
writer.Clear(30); // Password (empty)
|
||||
}
|
||||
}
|
||||
|
|
@ -276,7 +276,7 @@ public static class OutgoingAccountPackets
|
|||
else
|
||||
{
|
||||
var name = (m.RawName?.Trim()).DefaultIfNullOrEmpty("-no name-");
|
||||
writer.WriteAscii(name, 30);
|
||||
writer.WriteLatin1(name, 30);
|
||||
writer.Clear(30); // password
|
||||
}
|
||||
}
|
||||
|
|
@ -341,7 +341,7 @@ public static class OutgoingAccountPackets
|
|||
else
|
||||
{
|
||||
var name = (m.RawName?.Trim()).DefaultIfNullOrEmpty("-no name-");
|
||||
writer.WriteAscii(name, 30);
|
||||
writer.WriteLatin1(name, 30);
|
||||
writer.Clear(30); // password
|
||||
}
|
||||
}
|
||||
|
|
@ -353,8 +353,8 @@ public static class OutgoingAccountPackets
|
|||
var ci = cityInfo[i];
|
||||
|
||||
writer.Write((byte)i);
|
||||
writer.WriteAscii(ci.City, textLength);
|
||||
writer.WriteAscii(ci.Building, textLength);
|
||||
writer.WriteLatin1(ci.City, textLength);
|
||||
writer.WriteLatin1(ci.Building, textLength);
|
||||
if (client70130)
|
||||
{
|
||||
writer.Write(ci.X);
|
||||
|
|
@ -428,7 +428,7 @@ public static class OutgoingAccountPackets
|
|||
var si = info[i];
|
||||
|
||||
writer.Write((ushort)i);
|
||||
writer.WriteAscii(si.Name, 32);
|
||||
writer.WriteLatin1(si.Name, 32);
|
||||
writer.Write((byte)si.FullPercent);
|
||||
writer.Write((sbyte)si.TimeZone);
|
||||
// UO only supports IPv4
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ public static class OutgoingEquipmentPackets
|
|||
writer.Write(-3); // crafted by
|
||||
|
||||
writer.Write((ushort)crafterName.Length);
|
||||
writer.WriteAscii(crafterName);
|
||||
writer.WriteLatin1(crafterName);
|
||||
}
|
||||
|
||||
if (unidentified)
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ public static class OutgoingMenuPackets
|
|||
|
||||
if (question != null)
|
||||
{
|
||||
writer.WriteAscii(question);
|
||||
writer.WriteLatin1(question);
|
||||
}
|
||||
|
||||
writer.Write((byte)entriesLength);
|
||||
|
|
@ -84,7 +84,7 @@ public static class OutgoingMenuPackets
|
|||
{
|
||||
var nameLength = name.Length;
|
||||
writer.Write((byte)nameLength);
|
||||
writer.WriteAscii(name);
|
||||
writer.WriteLatin1(name);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -120,7 +120,7 @@ public static class OutgoingMenuPackets
|
|||
|
||||
if (question != null)
|
||||
{
|
||||
writer.WriteAscii(question);
|
||||
writer.WriteLatin1(question);
|
||||
}
|
||||
|
||||
writer.Write((byte)answersLength);
|
||||
|
|
@ -139,7 +139,7 @@ public static class OutgoingMenuPackets
|
|||
{
|
||||
var nameLength = answer.Length;
|
||||
writer.Write((byte)nameLength);
|
||||
writer.WriteAscii(answer);
|
||||
writer.WriteLatin1(answer);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ public static class OutgoingMessagePackets
|
|||
writer.Write((short)hue);
|
||||
writer.Write((short)font);
|
||||
writer.Write(number);
|
||||
writer.WriteAscii(name, 30);
|
||||
writer.WriteLatin1(name, 30);
|
||||
writer.WriteLittleUniNull(args);
|
||||
|
||||
writer.WritePacketLength();
|
||||
|
|
@ -139,8 +139,8 @@ public static class OutgoingMessagePackets
|
|||
writer.Write((short)font);
|
||||
writer.Write(number);
|
||||
writer.Write((byte)affixType);
|
||||
writer.WriteAscii(name, 30);
|
||||
writer.WriteAsciiNull(affix);
|
||||
writer.WriteLatin1(name, 30);
|
||||
writer.WriteLatin1Null(affix);
|
||||
writer.WriteBigUniNull(args);
|
||||
|
||||
writer.WritePacketLength();
|
||||
|
|
@ -214,13 +214,13 @@ public static class OutgoingMessagePackets
|
|||
writer.Write((short)font);
|
||||
if (ascii)
|
||||
{
|
||||
writer.WriteAscii(name, 30);
|
||||
writer.WriteAsciiNull(text);
|
||||
writer.WriteLatin1(name, 30);
|
||||
writer.WriteLatin1Null(text);
|
||||
}
|
||||
else
|
||||
{
|
||||
writer.WriteAscii(lang, 4);
|
||||
writer.WriteAscii(name, 30);
|
||||
writer.WriteLatin1(name, 30);
|
||||
writer.WriteBigUniNull(text);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -309,7 +309,7 @@ public static class OutgoingMobilePackets
|
|||
writer.Write((byte)0x98); // Packet ID
|
||||
writer.Write((ushort)37);
|
||||
writer.Write(m.Serial);
|
||||
writer.WriteAscii(m.Name ?? "", 29);
|
||||
writer.WriteLatin1(m.Name ?? "", 29);
|
||||
writer.Write((byte)0); // Null terminator
|
||||
|
||||
ns.Send(writer.Span);
|
||||
|
|
@ -507,7 +507,7 @@ public static class OutgoingMobilePackets
|
|||
writer.Write((byte)0x11); // Packet ID
|
||||
writer.Seek(2, SeekOrigin.Current);
|
||||
writer.Write(beheld.Serial);
|
||||
writer.WriteAscii(name, 30);
|
||||
writer.WriteLatin1(name, 30);
|
||||
writer.WriteAttribute(beheld.HitsMax, beheld.Hits, version == 0, true);
|
||||
writer.Write(canBeRenamed);
|
||||
writer.Write((byte)version);
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ public static class OutgoingPlayerPackets
|
|||
writer.Write((byte)0xB8); // Packet ID
|
||||
writer.Write((ushort)length);
|
||||
writer.Write(m);
|
||||
writer.WriteAsciiNull(header);
|
||||
writer.WriteLatin1Null(header);
|
||||
writer.WriteBigUniNull(footer);
|
||||
writer.WriteBigUniNull(body);
|
||||
|
||||
|
|
@ -264,7 +264,7 @@ public static class OutgoingPlayerPackets
|
|||
var writer = new SpanWriter(stackalloc byte[66]);
|
||||
writer.Write((byte)0x88); // Packet ID
|
||||
writer.Write(m);
|
||||
writer.WriteAscii(title, 60);
|
||||
writer.WriteLatin1(title, 60);
|
||||
writer.Write(flags);
|
||||
|
||||
ns.Send(writer.Span);
|
||||
|
|
@ -302,7 +302,7 @@ public static class OutgoingPlayerPackets
|
|||
writer.Write((byte)type);
|
||||
writer.Write(tip);
|
||||
writer.Write((ushort)text.Length);
|
||||
writer.WriteAscii(text);
|
||||
writer.WriteLatin1(text);
|
||||
|
||||
ns.Send(writer.Span);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ public static class OutgoingSecureTradePackets
|
|||
writer.Write(second.Serial);
|
||||
writer.Write(true);
|
||||
|
||||
writer.WriteAscii(name ?? "", 30);
|
||||
writer.WriteLatin1(name ?? "", 30);
|
||||
|
||||
ns.Send(writer.Span);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ public static class OutgoingVendorBuyPackets
|
|||
var desc = bis.Description ?? "";
|
||||
|
||||
writer.Write((byte)(desc.Length + 1));
|
||||
writer.WriteAsciiNull(desc);
|
||||
writer.WriteLatin1Null(desc);
|
||||
}
|
||||
|
||||
ns.Send(writer.Span);
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ public static class OutgoingVendorSellPackets
|
|||
var name = (item.Name?.Trim()).DefaultIfNullOrEmpty(sis.Name ?? "");
|
||||
|
||||
writer.Write((ushort)name.Length);
|
||||
writer.WriteAscii(name);
|
||||
writer.WriteLatin1(name);
|
||||
}
|
||||
|
||||
writer.WritePacketLength();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue