fix: Adds BWT cliloc support for v7.0.104+ (#1982)

Projects/Server/Client/BwtDecompress.cs

@@ -0,0 +1,175 @@
+using System;
+using System.IO;
+using System.Numerics;
+using System.Runtime.InteropServices;
+
+namespace Server;
+
+public static class BwtDecompress
+{
+    public static byte[] Decompress(Stream stream, int length)
+    {
+        var firstChar = (byte)stream.ReadByte();
+
+        Span<ushort> table = GC.AllocateUninitializedArray<ushort>(256 * 256);
+        Span<byte> output = GC.AllocateUninitializedArray<byte>(length);
+        BuildTable(table, firstChar);
+
+        var i = 0;
+        while (stream.Position < stream.Length)
+        {
+            var currentValue = firstChar;
+            var value = table[currentValue];
+            if (currentValue > 0)
+            {
+                do
+                {
+                    table[currentValue] = table[currentValue - 1];
+                } while (--currentValue > 0);
+            }
+
+            table[0] = value;
+
+            output[i++] = (byte)value;
+            firstChar = (byte)stream.ReadByte();
+        }
+
+        return InternalDecompress(output);
+    }
+
+    private static void BuildTable(Span<ushort> table, byte startValue)
+    {
+        var index = 0;
+        var firstByte = startValue;
+        byte secondByte = 0;
+        for (var i = 0; i < 256 * 256; i++)
+        {
+            var val = (ushort)(firstByte + (secondByte << 8));
+            table[index++] = val;
+
+            firstByte++;
+            if (firstByte == 0)
+            {
+                secondByte++;
+            }
+        }
+
+        table.Sort();
+    }
+
+    private static byte[] InternalDecompress(Span<byte> input)
+    {
+        Span<byte> symbolTable = stackalloc byte[256];
+        Span<byte> frequency = stackalloc byte[256];
+        Span<int> partialInput = stackalloc int[256 * 3];
+
+        for (var i = 0; i < 256; i++)
+        {
+            symbolTable[i] = (byte)i;
+        }
+
+        MemoryMarshal.Cast<byte, int>(input)[..256].CopyTo(partialInput);
+
+        var sum = 0;
+        for (var i = 0; i < 256; i++)
+        {
+            sum += partialInput[i];
+        }
+
+        var nonZeroCount = 256 - partialInput[..256].Count(0);
+
+        Frequency(partialInput, frequency);
+
+        for (int i = 0, m = 0; i < nonZeroCount; ++i)
+        {
+            var freq = frequency[i];
+            symbolTable[input[m + 1024]] = freq;
+            partialInput[freq + 256] = m + 1;
+            m += partialInput[freq];
+            partialInput[freq + 512] = m;
+        }
+
+        var val = symbolTable[0];
+        var output = GC.AllocateUninitializedArray<byte>(sum);
+
+        var count = 0;
+        do
+        {
+            ref var firstValRef = ref partialInput[val + 256];
+            output[count] = val;
+
+            if (firstValRef >= partialInput[val + 512])
+            {
+                if (nonZeroCount-- > 0)
+                {
+                    ShiftLeftSimd(symbolTable, nonZeroCount);
+                    val = symbolTable[0];
+                }
+            }
+            else
+            {
+                var idx = input[firstValRef + 1024];
+                firstValRef++;
+
+                if (idx != 0)
+                {
+                    ShiftLeftSimd(symbolTable, idx);
+                    symbolTable[idx] = val;
+                    val = symbolTable[0];
+                }
+            }
+
+            count++;
+        } while (count < sum);
+
+        return output;
+    }
+
+    private static void Frequency(Span<int> input, Span<byte> output)
+    {
+        Span<int> tmp = stackalloc int[256];
+        input[..256].CopyTo(tmp);
+
+        for (var i = 0; i < 256; i++)
+        {
+            uint value = 0;
+            byte index = 0;
+
+            for (var j = 0; j < 256; j++)
+            {
+                if (tmp[j] > value)
+                {
+                    index = (byte)j;
+                    value = (uint)tmp[j];
+                }
+            }
+
+            if (value == 0)
+            {
+                break;
+            }
+
+            output[i] = index;
+            tmp[index] = 0;
+        }
+    }
+
+    private static void ShiftLeftSimd(Span<byte> input, int max)
+    {
+        var i = 0;
+        var vectorSize = Vector<byte>.Count;
+
+        while (i + vectorSize <= max)
+        {
+            var vector = new Vector<byte>(input[(i + 1)..]);
+            vector.CopyTo(input[i..]);
+            i += vectorSize;
+        }
+
+        while (i < max)
+        {
+            input[i] = input[i + 1];
+            i++;
+        }
+    }
+}

Projects/Server/Localization/Localization.cs

@@ -14,6 +14,7 @@
  *************************************************************************/
 
 using System;
+using System.Buffers.Binary;
 using System.Collections.Generic;
 using System.IO;
 using System.Runtime.CompilerServices;
@@ -96,28 +97,46 @@ public static class Localization
         if (File.Exists(file))
         {
             using var fs = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read);
-            using var bin = new BinaryReader(fs);
+            Span<byte> header = stackalloc byte[6];
+            fs.Read(header);
 
-            bin.ReadInt32();
+            byte[] data;
-            bin.ReadInt16();
+            BufferReader br;
+            if (BinaryPrimitives.ReadInt32LittleEndian(header) != 2 || BinaryPrimitives.ReadInt16LittleEndian(header[4..]) != 1)
+            {
+                // Skip header
+                fs.Position = 4;
+                data = BwtDecompress.Decompress(fs, (int)fs.Length - 4);
+                br = new BufferReader(data);
+
+                var header2 = br.ReadInt();   // Header 2
+                var header1 = br.ReadShort(); // Header 1
+
+                if (header2 != 2 || header1 != 1)
+                {
+                    throw new Exception($"Invalid cliloc header in {file}");
+                }
+            }
+            else
+            {
+                data = GC.AllocateUninitializedArray<byte>((int)fs.Length - 6);
+                fs.Read(data);
+                br = new BufferReader(data);
+            }
 
             byte[] buffer = null;
-            while (bin.BaseStream.Length != bin.BaseStream.Position)
+            while (br.Position < data.Length)
             {
-                var number = bin.ReadInt32();
+                var number = br.ReadInt();
-                var flag = bin.ReadByte(); // Original, Custom, Modified
+                var flag = br.ReadByte(); // Original, Custom, Modified
-                var length = bin.ReadInt16();
+                var length = br.ReadShort();
 
                 if (buffer == null || buffer.Length < length)
                 {
                     buffer = GC.AllocateUninitializedArray<byte>(length);
                 }
 
-                var bytesRead = bin.Read(buffer, 0, length);
+                br.Read(buffer.AsSpan(0, length));
-                if (bytesRead != length)
-                {
-                    throw new Exception($"Could not read enough bytes from {file}");
-                }
 
                 var text = Encoding.UTF8.GetString(buffer.AsSpan(0, length));
                 entries[number] = new LocalizationEntry(lang, number, text);