From c265bcbb5eeacd229982cf34b0b6d6975e39502b Mon Sep 17 00:00:00 2001
From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com>
Date: Sun, 7 Jun 2026 00:22:11 -0700
Subject: [PATCH] feat(pathfinding): .swb format v7 per-chunk compression (#3a)
(#2470)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## Summary
Phase #3a, stacked on #2469. Compresses each chunk record independently with libdeflate (random access preserved).
Trammel: 124.7 MB → 19.2 MB (−85%).
## Details
- Whole-record framing `[u32 UncompressedLen][payload]`; records that don't shrink (tiny Uniform) are stored raw, detected as payload length == UncompressedLen.
- Codec chosen by full-Trammel spike: libdeflate VeryHigh (16.5 MB, 1.83 µs/chunk decompress) over zstd L19/22 (17.4 MB) and managed Brotli q11 (16.4 MB) — best native ratio, fastest decompress, already the repo's packet codec (no new dependency).
- Reuses cached thread-static bindings: `Deflate.Maximum` for bake, `Deflate.Standard` for reads.
- Compression is bake-time only; decompression is one-time per chunk (LRU-cached).
- Format v7; v6 files rejected and re-baked once.
## Tests
v7 unit tests + the v6 suite run through the compression path; full pathfinding suite green; Release build clean.
---
Projects/Server/Compression/Deflate.cs | 8 ++
.../Engines/Pathing/StepCacheFileV7Tests.cs | 124 ++++++++++++++++++
.../Engines/Pathing/Cache/StepCacheFile.cs | 114 ++++++++++++++--
dev-docs/pathfinding.md | 24 +++-
4 files changed, 255 insertions(+), 15 deletions(-)
create mode 100644 Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV7Tests.cs
diff --git a/Projects/Server/Compression/Deflate.cs b/Projects/Server/Compression/Deflate.cs
index fbe45331d..556c41e6c 100644
--- a/Projects/Server/Compression/Deflate.cs
+++ b/Projects/Server/Compression/Deflate.cs
@@ -23,5 +23,13 @@ public static class Deflate
[ThreadStatic]
private static LibDeflateBinding _standard;
+ [ThreadStatic]
+ private static LibDeflateBinding _maximum;
+
public static LibDeflateBinding Standard => _standard ??= new LibDeflateBinding();
+
+ // Best-ratio compressor. Construction allocates a native libdeflate compressor, so it is
+ // cached per thread like Standard and reused. Decompression is level-independent, so the
+ // decompress path can use either accessor. libdeflate is not thread-safe — hence ThreadStatic.
+ public static LibDeflateBinding Maximum => _maximum ??= new LibDeflateBinding(LibDeflateCompressionLevel.VeryHigh);
}
diff --git a/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV7Tests.cs b/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV7Tests.cs
new file mode 100644
index 000000000..dc42c71da
--- /dev/null
+++ b/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV7Tests.cs
@@ -0,0 +1,124 @@
+using System;
+using System.IO;
+using Server.Engines.Pathing.Cache;
+using Xunit;
+
+namespace Server.Tests.Pathfinding;
+
+// v7 = per-chunk libdeflate compression on top of the v6 predictive-Z format. Each record is
+// compressed independently (random access preserved) behind a u32 uncompressed-length prefix;
+// records that do not shrink (tiny Uniform records) are stored raw. StepCacheFileV6Tests already
+// round-trips every chunk shape through this compression path; these tests cover the v7-specific
+// behavior: that compression engages, that the stored-raw fallback round-trips, and v6 rejection.
+[Collection("Sequential Pathfinding Tests")]
+public class StepCacheFileV7Tests
+{
+ private static StepChunk VariedChunk(int multis = 3)
+ {
+ var c = new StepChunk { BuiltMultisVersion = multis };
+ for (var i = 0; i < StepChunk.CellsPerChunk; i++)
+ {
+ c.WalkMask[i] = (byte)(i & 0xFF);
+ c.WetMask[i] = (byte)((i * 7) & 0xFF);
+ c.SourceZ[i] = (sbyte)((i % 40) - 20);
+ c.WalkZN[i] = (sbyte)(c.SourceZ[i] + (i % 3));
+ c.SwimZS[i] = (sbyte)(c.SourceZ[i] - (i % 2));
+ }
+ return c;
+ }
+
+ private static StepChunk UniformChunk(byte walk = 0xC1, sbyte z = 10, int multis = 7)
+ {
+ var c = new StepChunk { BuiltMultisVersion = multis };
+ Array.Fill(c.WalkMask, walk);
+ Array.Fill(c.SourceZ, z);
+ foreach (var arr in new[]
+ {
+ c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW,
+ c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW
+ })
+ {
+ Array.Fill(arr, z);
+ }
+ return c;
+ }
+
+ private static string Write1(StepChunk c, int cx, int cy)
+ {
+ var path = Path.Combine(Path.GetTempPath(), $"swbv7_{Guid.NewGuid():N}.swb");
+ var emitted = false;
+ StepCacheFile.Write(path, 1u, 1u, (out int ox, out int oy, out StepChunk oc) =>
+ {
+ if (emitted) { ox = oy = 0; oc = null!; return false; }
+ emitted = true; ox = cx; oy = cy; oc = c; return true;
+ });
+ return path;
+ }
+
+ private static StepChunk RoundTrip(StepChunk src, int cx, int cy, out long fileLen)
+ {
+ var path = Write1(src, cx, cy);
+ try
+ {
+ fileLen = new FileInfo(path).Length;
+ using var reader = StepCacheFile.OpenForLazy(path);
+ Assert.NotNull(reader);
+ var rt = reader!.TryReadChunk(cx, cy);
+ Assert.NotNull(rt);
+ return rt!;
+ }
+ finally { File.Delete(path); }
+ }
+
+ private static void AssertBaseEqual(StepChunk a, StepChunk b)
+ {
+ Assert.Equal(a.BuiltMultisVersion, b.BuiltMultisVersion);
+ Assert.True(a.WalkMask.AsSpan().SequenceEqual(b.WalkMask));
+ Assert.True(a.WetMask.AsSpan().SequenceEqual(b.WetMask));
+ Assert.True(a.SourceZ.AsSpan().SequenceEqual(b.SourceZ));
+ var az = new[] { a.WalkZN, a.WalkZNE, a.WalkZE, a.WalkZSE, a.WalkZS, a.WalkZSW, a.WalkZW, a.WalkZNW,
+ a.SwimZN, a.SwimZNE, a.SwimZE, a.SwimZSE, a.SwimZS, a.SwimZSW, a.SwimZW, a.SwimZNW };
+ var bz = new[] { b.WalkZN, b.WalkZNE, b.WalkZE, b.WalkZSE, b.WalkZS, b.WalkZSW, b.WalkZW, b.WalkZNW,
+ b.SwimZN, b.SwimZNE, b.SwimZE, b.SwimZSE, b.SwimZS, b.SwimZSW, b.SwimZW, b.SwimZNW };
+ for (var i = 0; i < az.Length; i++)
+ {
+ Assert.True(az[i].AsSpan().SequenceEqual(bz[i]), $"base Z array {i} differs");
+ }
+ }
+
+ [Fact]
+ public void Varied_Compresses_AndRoundTrips()
+ {
+ var src = VariedChunk(multis: 4);
+ var rt = RoundTrip(src, 1, 2, out var fileLen);
+ AssertBaseEqual(src, rt);
+ // The uncompressed v6 Full record for a varied chunk is > 5 KB. Compressed + header(48)
+ // + index(20), the whole file must be well under that — proving compression engaged.
+ Assert.True(fileLen < 4000, $"expected compression to shrink the record; file was {fileLen} bytes");
+ }
+
+ [Fact]
+ public void Uniform_StoredRaw_RoundTrips()
+ {
+ // A Uniform record body is ~24 bytes; libdeflate cannot shrink it, so WriteChunk stores it
+ // raw (payload length == uncompressed length). The reader must take the raw path and rebuild.
+ var src = UniformChunk(walk: 0xC1, z: 12, multis: 9);
+ var rt = RoundTrip(src, 5, 6, out var fileLen);
+ AssertBaseEqual(src, rt);
+ Assert.True(fileLen < 200, $"uniform record should stay tiny; file was {fileLen} bytes");
+ }
+
+ [Fact]
+ public void V6_IsRejected()
+ {
+ var path = Write1(UniformChunk(), 0, 0);
+ try
+ {
+ var bytes = File.ReadAllBytes(path);
+ bytes[4] = 6; bytes[5] = 0; bytes[6] = 0; bytes[7] = 0; // version 6 < MinSupportedVersion 7
+ File.WriteAllBytes(path, bytes);
+ Assert.Null(StepCacheFile.OpenForLazy(path));
+ }
+ finally { File.Delete(path); }
+ }
+}
diff --git a/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs b/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs
index c34412604..a578650f7 100644
--- a/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs
+++ b/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs
@@ -2,7 +2,9 @@ using System;
using System.Buffers.Binary;
using System.Collections.Generic;
using System.IO;
+using System.IO.Compression;
using System.Runtime.InteropServices;
+using Server.Compression;
namespace Server.Engines.Pathing.Cache;
@@ -14,11 +16,11 @@ namespace Server.Engines.Pathing.Cache;
/// only when the cache asks for them. RAM stays bounded by MaxResidentChunks regardless
/// of file size.
///
-/// File layout v6 (little-endian, BufferWriter / BufferReader convention):
+/// File layout v7 (little-endian, BufferWriter / BufferReader convention):
///
/// Header (48 bytes):
/// u32 Magic = 0x42575300 ('SWB\0')
-/// u32 Version = current FormatVersion (6)
+/// u32 Version = current FormatVersion (7)
/// u32 MapId
/// u64 Fingerprint XxHash3 over (1) LandTable + ItemTable flags AND (2) the
/// on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul.
@@ -31,6 +33,13 @@ namespace Server.Engines.Pathing.Cache;
/// u64 IndexOffset File position where the chunk index begins.
///
/// Per chunk (ChunkCount times, variable size):
+/// u32 UncompressedLen Size of the inflated record body below.
+/// byte[] Payload The record body (the v6 layout that follows), libdeflate-
+/// compressed. If the on-disk payload length (index recordLength −
+/// 4) equals UncompressedLen, the body was stored raw because
+/// compression did not shrink it (tiny Uniform records).
+///
+/// Record body (after inflate — the v6 layout):
/// u16 ChunkX
/// u16 ChunkY
/// u32 BuiltMultisVersion
@@ -79,7 +88,7 @@ namespace Server.Engines.Pathing.Cache;
internal static class StepCacheFile
{
public const uint Magic = 0x42575300; // 'SWB\0'
- public const uint FormatVersion = 6;
+ public const uint FormatVersion = 7;
///
/// Lowest format version this binary can load. Files below this version are treated as
@@ -97,8 +106,11 @@ internal static class StepCacheFile
/// predictive-Z residuals: each base directional Z array is stored as a masked residual
/// against SourceZ (a ZArrayMask u16 flags which arrays are present); arrays matching their
/// prediction are omitted and synthesized at read. v5 files are rejected and re-baked once.
+ /// Bumped to 7 for per-chunk compression: each chunk record is libdeflate-compressed
+ /// independently (random access preserved) behind a u32 uncompressed-length prefix; tiny
+ /// records that do not shrink are stored raw. v6 files are rejected and re-baked once.
///
- public const uint MinSupportedVersion = 6;
+ public const uint MinSupportedVersion = 7;
// Per-chunk record discriminator (first byte after BuiltMultisVersion). 1 is reserved.
private const byte KindFull = 0;
@@ -255,6 +267,15 @@ internal static class StepCacheFile
w.Write(chunkCount);
w.Write(0UL); // IndexOffset placeholder, patched after chunks
+ // Per-chunk compression: each record is built uncompressed into `recordScratch`, then
+ // libdeflate-compressed into `compScratch` and framed as [u32 uncompressedLen][payload].
+ // Reuse the cached per-thread VeryHigh compressor (construction allocates native state).
+ // libdeflate is not thread-safe, but bake runs single-threaded. VeryHigh is the best-ratio
+ // level and its slow compression is irrelevant offline (decompression is what's hot, ~1.8us).
+ var packer = Deflate.Maximum;
+ var recordScratch = new byte[BytesPerChunkBase + 1024];
+ var compScratch = new byte[packer.MaxPackSize(recordScratch.Length)];
+
var indexEntries = new (ulong key, ulong offset, uint length)[chunkCount];
var written = 0u;
while (next(out var chunkX, out var chunkY, out var chunk))
@@ -266,7 +287,7 @@ internal static class StepCacheFile
);
}
var chunkOffset = (ulong)w.Position;
- WriteChunk(w, chunkX, chunkY, chunk);
+ WriteChunk(w, chunkX, chunkY, chunk, packer, ref recordScratch, ref compScratch);
var chunkLength = (uint)((ulong)w.Position - chunkOffset);
indexEntries[written] = (PackChunkKey(chunkX, chunkY), chunkOffset, chunkLength);
written++;
@@ -412,7 +433,44 @@ internal static class StepCacheFile
c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW,
};
- private static void WriteChunk(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
+ ///
+ /// Builds the uncompressed v6 record for one chunk into , libdeflate-
+ /// compresses it, and writes it framed as [u32 uncompressedLen][payload]. The payload is the
+ /// compressed bytes, or — when compression does not shrink the record (tiny Uniform records) —
+ /// the raw record itself; the reader distinguishes the two by payload length vs uncompressedLen.
+ ///
+ private static void WriteChunk(
+ BufferWriter w, int chunkX, int chunkY, StepChunk chunk,
+ LibDeflateBinding packer, ref byte[] recordScratch, ref byte[] compScratch
+ )
+ {
+ var rw = new BufferWriter(recordScratch, prefixStr: false);
+ BuildRecord(rw, chunkX, chunkY, chunk);
+ recordScratch = rw.Buffer; // may have grown; keep the larger buffer for reuse
+ var recordLen = (int)rw.Position;
+
+ var bound = packer.MaxPackSize(recordLen);
+ if (compScratch.Length < bound)
+ {
+ compScratch = new byte[bound];
+ }
+
+ var compLen = packer.Pack(compScratch, recordScratch.AsSpan(0, recordLen));
+
+ w.Write((uint)recordLen);
+ if (compLen > 0 && compLen < recordLen)
+ {
+ w.Write(compScratch.AsSpan(0, compLen));
+ }
+ else
+ {
+ // Incompressible (or expanded): store the record raw. The reader detects this when
+ // the on-disk payload length equals the uncompressed length.
+ w.Write(recordScratch.AsSpan(0, recordLen));
+ }
+ }
+
+ private static void BuildRecord(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
{
w.Write((ushort)chunkX);
w.Write((ushort)chunkY);
@@ -644,7 +702,8 @@ internal static class StepCacheFile
{
private FileStream _stream;
private readonly Dictionary _offsets;
- private byte[] _buffer;
+ private byte[] _buffer; // raw on-disk record: [u32 uncompressedLen][payload]
+ private byte[] _bodyBuffer; // decompressed v6 record, parsed by ReadChunk
public uint MapId { get; }
public ulong Fingerprint { get; }
@@ -679,6 +738,7 @@ internal static class StepCacheFile
ChunkCount = chunkCount;
_offsets = offsets;
_buffer = new byte[BytesPerChunkBase];
+ _bodyBuffer = new byte[BytesPerChunkBase];
}
///
@@ -699,8 +759,7 @@ internal static class StepCacheFile
return null;
}
- // Grow the scratch buffer if this chunk's record is larger than what we have.
- // Common case: chunks fit in BytesPerChunkBase; only multi-Z-heavy chunks grow.
+ // Grow the on-disk scratch buffer if this chunk's record is larger than what we have.
if (entry.length > _buffer.Length)
{
_buffer = new byte[entry.length];
@@ -708,7 +767,41 @@ internal static class StepCacheFile
_stream.Position = (long)entry.offset;
var read = _stream.Read(_buffer, 0, (int)entry.length);
- return read < (int)entry.length ? null : ReadChunk(_buffer);
+ if (read < (int)entry.length || entry.length < sizeof(uint))
+ {
+ return null;
+ }
+
+ // Frame: [u32 uncompressedLen][payload]. payload is libdeflate-compressed, unless its
+ // length equals uncompressedLen, in which case it was stored raw (incompressible).
+ var uncompressedLen = (int)BinaryPrimitives.ReadUInt32LittleEndian(_buffer);
+ var payloadLen = (int)entry.length - sizeof(uint);
+ if (_bodyBuffer.Length < uncompressedLen)
+ {
+ _bodyBuffer = new byte[uncompressedLen];
+ }
+
+ if (payloadLen == uncompressedLen)
+ {
+ Array.Copy(_buffer, sizeof(uint), _bodyBuffer, 0, uncompressedLen);
+ }
+ else
+ {
+ // Decompression is level-independent, so reuse the shared per-thread binding
+ // rather than allocating a native decompressor per reader. The cache is read on
+ // the single game thread; libdeflate's non-thread-safety is satisfied by ThreadStatic.
+ var result = Deflate.Standard.Unpack(
+ _bodyBuffer.AsSpan(0, uncompressedLen),
+ _buffer.AsSpan(sizeof(uint), payloadLen),
+ out var produced
+ );
+ if (result != LibDeflateResult.Success || produced != uncompressedLen)
+ {
+ return null;
+ }
+ }
+
+ return ReadChunk(_bodyBuffer);
}
public void Dispose()
@@ -716,6 +809,7 @@ internal static class StepCacheFile
_stream?.Dispose();
_stream = null;
_buffer = null;
+ _bodyBuffer = null;
}
}
}
diff --git a/dev-docs/pathfinding.md b/dev-docs/pathfinding.md
index 0530dceb8..f9696b246 100644
--- a/dev-docs/pathfinding.md
+++ b/dev-docs/pathfinding.md
@@ -221,11 +221,20 @@ whole-file) and bounded RAM (only touched chunks materialize, LRU-capped):
saves ~0 bytes (a residual is still 1 byte/cell); the win is the per-array elision, and leaving
non-elided arrays in residual form makes #3 a pure codec add (no further transform/format bump).
Swim-layer + strata trailers stay absolute, deferred to #3.
-3. **Per-chunk block compression + index compaction.** zstd/deflate each chunk record
- independently (index already carries a per-chunk `length`; reader decompresses one chunk on
- read). At 256-cell granularity the **index overhead** then dominates (20 B/chunk ×
- 114 K ≈ 2.3 MB), so compact it in the same pass: chunks in fixed sweep order → implicit keys,
- delta-encoded offsets, lengths derivable → ~2–4 B/chunk.
+3. **Per-chunk block compression + index compaction.**
+ - **#3a — per-chunk compression: *shipped as format v7.*** Each record is libdeflate-compressed
+ independently (random access preserved; the reader inflates one chunk on read into a reused
+ buffer) behind a `u32 UncompressedLen` frame; tiny Uniform records that do not shrink are
+ stored raw (detected as on-disk payload length == UncompressedLen). Codec chosen by full-Trammel
+ spike: **libdeflate VeryHigh** beat zstd L19/22 (17.4 MB) and tied managed Brotli q11 (16.4 MB)
+ at **16.5 MB of compressed records**, with the *fastest* decompress (**1.83 µs/chunk**, vs zstd
+ 2.21) — and it is already the repo's packet codec (`LibDeflate.Bindings`), so no new dependency.
+ zstd's large-window advantage doesn't apply at 256-cell record granularity. Compression runs at
+ bake time only (offline); decompression is one-time per chunk (LRU-cached after). **Calibrated:
+ v6 124.7 MB → v7 19.2 MB (−85%); −97% vs the original 565 MB.**
+ - **#3b — index compaction (remaining, ~2 MB).** The v7 file's residual is the still-uncompacted
+ index (20 B/chunk × 114 K ≈ 2.3 MB). Compact it: chunks in fixed sweep order → implicit keys,
+ delta-encoded offsets, lengths derivable → ~3 B/chunk ≈ 0.34 MB, landing the file at ~17 MB.
**Do first:** a uniformity audit over a baked facet (how many chunks are fully uniform / have
all-zero Z residuals?) to size the #1/#2 win before writing any format code. Each technique is a
@@ -251,3 +260,8 @@ Validate size **and** read-latency vs the corpus in the benchmark repo after eac
by present walk-residual blocks (~46 MB, mostly ±1..3 + zeros → highly compressible), the
per-Full-chunk mask/SourceZ base (~33 MB), and absolute swim-layer trailers (~26 MB) — all prime
targets for #3 per-chunk compression.
+- *Calibrated v7 (`BakeMap`-measured, full Trammel):* **#3a per-chunk libdeflate VeryHigh: 124.7 MB
+ → 19.2 MB (−85%); −97% vs the original 565 MB.** Codec spike (per-chunk over the 122.4 MB of v6
+ records): libdeflate VeryHigh **16.5 MB** vs Brotli q11 16.4 / zstd L19–22 17.4; decompress
+ **1.83 µs/chunk** (libdeflate) vs 2.21 (zstd). The remaining ~2.3 MB is the uncompacted index
+ (→ #3b).