From c7697e1dc5d2a617c46c14cf276e5755a06615d9 Mon Sep 17 00:00:00 2001 From: Kamron Batman <3953314+kamronbatman@users.noreply.github.com> Date: Sat, 6 Jun 2026 15:20:27 -0700 Subject: [PATCH] =?UTF-8?q?feat(pathfinding):=20.swb=20uniform-chunk=20eli?= =?UTF-8?q?sion=20(format=20v5)=20=E2=80=94=20Trammel=20592=E2=86=92232=20?= =?UTF-8?q?MB=20(#2465)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Sub-project #1 of the `.swb` step-cache size-reduction roadmap (`dev-docs/pathfinding.md` § Future work). Adds **uniform-chunk elision** to the `StepCacheFile` format, bumping it **v4 → v5**. A fully-uniform 16×16 chunk — no strata, **no swim layer**, all 19 base arrays constant (open ocean, Green Acres, void) — serializes to a **~28-byte record** (`KindUniform`) instead of ~5,393, and reconstructs **byte-identically** via `Array.Fill`. Non-uniform chunks use the existing v4 body (`KindFull`) with the swim-layer and strata trailers **fully preserved** — the Kind byte is just prepended. ## Calibrated result (measured, not projected) Baked Trammel via `SaveToFile`: | | | |---|---:| | Chunks | 114,688 | | Uniform (swim-aware) → elided | 62.7% | | Swim-layer chunks (stay Full) | 8.9% | | Strata chunks (stay Full) | 1.8% | | Baseline (full records) | 592.2 MB | | **Actual v5 `.swb`** | **231.9 MB (−61%)** | The residual is ~150 MB of non-uniform land Z-blocks (targeted by #2 predictive-Z) + ~81 MB of swim-layer trailers (#2/#3). #2 and #3 are separate follow-up PRs. ## Implementation - `StepChunk.IsUniform()` — false if it has strata **or a swim layer**, else true only when all 19 base arrays are constant (the "all-same" check uses the SIMD-accelerated `ContainsAnyExcept`). - `StepCacheFile` v5 — `Kind` byte (`KindFull=0`/`KindUniform=2`, 1 reserved); uniform write/read; `FormatVersion`/`MinSupportedVersion` → 5 (v4 files rejected on open and re-baked). No `StepCache`/algorithm/index changes; fingerprint logic untouched. ## Tests 7 `StepCacheFileV5Tests` (uniform round-trip + `<200 B` compactness, varied-full, swim-layer-full, strata-full, swim+strata combined, v4 version-gate rejection) + the existing StepCache/pathfinding suite — **70 pass**, including the prior `SwimLayer_RoundTrips`. An independent review verified write/read symmetry, cast round-tripping, swim/strata preservation, and the version gate (READY TO MERGE). --- .../Engines/Pathing/StepCacheFileV5Tests.cs | 213 ++++++++++++++++++ .../Engines/Pathing/Cache/StepCacheFile.cs | 82 ++++++- .../Engines/Pathing/Cache/StepChunk.cs | 25 ++ dev-docs/pathfinding.md | 13 +- 4 files changed, 324 insertions(+), 9 deletions(-) create mode 100644 Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV5Tests.cs diff --git a/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV5Tests.cs b/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV5Tests.cs new file mode 100644 index 000000000..32ce16d10 --- /dev/null +++ b/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV5Tests.cs @@ -0,0 +1,213 @@ +using System; +using System.IO; +using Server.Engines.Pathing.Cache; +using Xunit; + +namespace Server.Tests.Pathfinding; + +// v5 = uniform-chunk elision on top of the v4 swim-layer format. A uniform chunk (no strata, +// no swim layer, all 19 base arrays constant) serializes to ~28 bytes; Full chunks (incl. swim +// layer + strata) round-trip byte-identically. +[Collection("Sequential Pathfinding Tests")] +public class StepCacheFileV5Tests +{ + private static StepChunk UniformChunk(byte walk = 0xC1, byte wet = 0x00, sbyte z = 10, int multis = 7) + { + var c = new StepChunk { BuiltMultisVersion = multis }; + Array.Fill(c.WalkMask, walk); + Array.Fill(c.WetMask, wet); + Array.Fill(c.SourceZ, z); + foreach (var arr in new[] + { + c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW, + c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW + }) + { + Array.Fill(arr, z); + } + return c; + } + + private static StepChunk VariedChunk(int multis = 3) + { + var c = new StepChunk { BuiltMultisVersion = multis }; + for (var i = 0; i < StepChunk.CellsPerChunk; i++) + { + c.WalkMask[i] = (byte)(i & 0xFF); + c.WetMask[i] = (byte)((i * 7) & 0xFF); + c.SourceZ[i] = (sbyte)((i % 40) - 20); + c.WalkZN[i] = (sbyte)(c.SourceZ[i] + (i % 3)); + c.SwimZS[i] = (sbyte)(c.SourceZ[i] - (i % 2)); + } + return c; + } + + private static StepChunk SwimChunk(int multis = 6) + { + var c = VariedChunk(multis); + c.AllocateSwimLayer(); + for (var i = 0; i < StepChunk.CellsPerChunk; i++) + { + c.SwimSourceZ[i] = (sbyte)((i % 30) - 15); + c.SwimMask[i] = (byte)((i * 5) & 0xFF); + c.SwimZN_Layer[i] = (sbyte)(i % 7); + c.SwimZNW_Layer[i] = (sbyte)(-(i % 4)); + } + return c; + } + + private static void AssertChunksEqual(StepChunk a, StepChunk b) + { + Assert.Equal(a.BuiltMultisVersion, b.BuiltMultisVersion); + Assert.True(a.WalkMask.AsSpan().SequenceEqual(b.WalkMask)); + Assert.True(a.WetMask.AsSpan().SequenceEqual(b.WetMask)); + Assert.True(a.SourceZ.AsSpan().SequenceEqual(b.SourceZ)); + + var az = new[] { a.WalkZN, a.WalkZNE, a.WalkZE, a.WalkZSE, a.WalkZS, a.WalkZSW, a.WalkZW, a.WalkZNW, + a.SwimZN, a.SwimZNE, a.SwimZE, a.SwimZSE, a.SwimZS, a.SwimZSW, a.SwimZW, a.SwimZNW }; + var bz = new[] { b.WalkZN, b.WalkZNE, b.WalkZE, b.WalkZSE, b.WalkZS, b.WalkZSW, b.WalkZW, b.WalkZNW, + b.SwimZN, b.SwimZNE, b.SwimZE, b.SwimZSE, b.SwimZS, b.SwimZSW, b.SwimZW, b.SwimZNW }; + for (var i = 0; i < az.Length; i++) + { + Assert.True(az[i].AsSpan().SequenceEqual(bz[i]), $"base Z array {i} differs"); + } + + Assert.Equal(a.HasSwimLayer, b.HasSwimLayer); + if (a.HasSwimLayer) + { + Assert.True(a.SwimSourceZ.AsSpan().SequenceEqual(b.SwimSourceZ)); + Assert.True(a.SwimMask.AsSpan().SequenceEqual(b.SwimMask)); + var al = new[] { a.SwimZN_Layer, a.SwimZNE_Layer, a.SwimZE_Layer, a.SwimZSE_Layer, + a.SwimZS_Layer, a.SwimZSW_Layer, a.SwimZW_Layer, a.SwimZNW_Layer }; + var bl = new[] { b.SwimZN_Layer, b.SwimZNE_Layer, b.SwimZE_Layer, b.SwimZSE_Layer, + b.SwimZS_Layer, b.SwimZSW_Layer, b.SwimZW_Layer, b.SwimZNW_Layer }; + for (var i = 0; i < al.Length; i++) + { + Assert.True(al[i].AsSpan().SequenceEqual(bl[i]), $"swim-layer Z array {i} differs"); + } + } + } + + private static string Write1(StepChunk c, int cx, int cy) + { + var path = Path.Combine(Path.GetTempPath(), $"swbv5_{Guid.NewGuid():N}.swb"); + var emitted = false; + StepCacheFile.Write(path, 1u, 1u, (out int ox, out int oy, out StepChunk oc) => + { + if (emitted) { ox = oy = 0; oc = null!; return false; } + emitted = true; ox = cx; oy = cy; oc = c; return true; + }); + return path; + } + + private static StepChunk RoundTrip(StepChunk src, int cx, int cy, out long fileLen) + { + var path = Write1(src, cx, cy); + try + { + fileLen = new FileInfo(path).Length; + using var reader = StepCacheFile.OpenForLazy(path); + Assert.NotNull(reader); + var rt = reader!.TryReadChunk(cx, cy); + Assert.NotNull(rt); + return rt!; + } + finally { File.Delete(path); } + } + + [Fact] + public void IsUniform_TrueForAllIdentical_FalseForVariedStrataOrSwim() + { + Assert.True(UniformChunk().IsUniform()); + + var varied = UniformChunk(); + varied.WalkZE[42] = 99; + Assert.False(varied.IsUniform()); + + var strata = UniformChunk(); + var offsets = new ushort[StepChunk.CellsPerChunk]; + Array.Fill(offsets, StepChunk.NoStrata); + offsets[0] = 0; + strata.SetStrata(offsets, new byte[] { 0 }); + Assert.False(strata.IsUniform()); + + var swim = UniformChunk(); + swim.AllocateSwimLayer(); // a uniform-looking base but with a swim layer is NOT uniform + Assert.False(swim.IsUniform()); + } + + [Fact] + public void Uniform_RoundTrips_Identically_AndIsCompact() + { + var src = UniformChunk(walk: 0xC1, wet: 0x00, z: 12, multis: 9); + var rt = RoundTrip(src, 5, 6, out var fileLen); + Assert.True(fileLen < 200, $"uniform .swb too large: {fileLen}"); + AssertChunksEqual(src, rt); + } + + [Fact] + public void Varied_Full_RoundTrips_Identically() + { + var src = VariedChunk(multis: 4); + AssertChunksEqual(src, RoundTrip(src, 1, 2, out _)); + } + + [Fact] + public void SwimLayer_Full_RoundTrips_Identically() + { + var src = SwimChunk(multis: 8); + var rt = RoundTrip(src, 7, 8, out _); + Assert.True(rt.HasSwimLayer); + AssertChunksEqual(src, rt); + } + + [Fact] + public void Strata_Full_RoundTrips_Identically() + { + var src = VariedChunk(multis: 5); + var offsets = new ushort[StepChunk.CellsPerChunk]; + Array.Fill(offsets, StepChunk.NoStrata); + offsets[10] = 0; + var data = new byte[1 + StepChunk.StratumByteLength]; + data[0] = 1; + src.SetStrata(offsets, data); + + var rt = RoundTrip(src, 3, 4, out _); + AssertChunksEqual(src, rt); + Assert.True(rt.IsCellMultiZ(10)); + Assert.True(rt.StrataData.SequenceEqual(src.StrataData)); + } + + [Fact] + public void OlderVersion_IsRejected() + { + var path = Write1(UniformChunk(), 0, 0); + try + { + var bytes = File.ReadAllBytes(path); + bytes[4] = 4; bytes[5] = 0; bytes[6] = 0; bytes[7] = 0; // version 4 < MinSupportedVersion 5 + File.WriteAllBytes(path, bytes); + Assert.Null(StepCacheFile.OpenForLazy(path)); + } + finally { File.Delete(path); } + } + + [Fact] + public void SwimAndStrata_Full_RoundTrips_Identically() + { + // Exercises the combined trailer ordering: swim-layer trailer THEN strata trailer. + var src = SwimChunk(multis: 11); + var offsets = new ushort[StepChunk.CellsPerChunk]; + Array.Fill(offsets, StepChunk.NoStrata); + offsets[20] = 0; + var data = new byte[1 + StepChunk.StratumByteLength]; + data[0] = 1; + src.SetStrata(offsets, data); + + var rt = RoundTrip(src, 9, 9, out _); + Assert.True(rt.HasSwimLayer); + Assert.True(rt.IsCellMultiZ(20)); + AssertChunksEqual(src, rt); + Assert.True(rt.StrataData.SequenceEqual(src.StrataData)); + } +} diff --git a/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs b/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs index cddfd4367..4e1df83bd 100644 --- a/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs +++ b/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs @@ -14,11 +14,11 @@ namespace Server.Engines.Pathing.Cache; /// only when the cache asks for them. RAM stays bounded by MaxResidentChunks regardless /// of file size. /// -/// File layout v3 (little-endian, BufferWriter / BufferReader convention): +/// File layout v5 (little-endian, BufferWriter / BufferReader convention): /// /// Header (48 bytes): /// u32 Magic = 0x42575300 ('SWB\0') -/// u32 Version = current FormatVersion (3) +/// u32 Version = current FormatVersion (5) /// u32 MapId /// u64 Fingerprint XxHash3 over (1) LandTable + ItemTable flags AND (2) the /// on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul. @@ -34,6 +34,10 @@ namespace Server.Engines.Pathing.Cache; /// u16 ChunkX /// u16 ChunkY /// u32 BuiltMultisVersion +/// u8 Kind 0 = Full; 2 = Uniform +/// // Uniform (Kind == 2): ~28-byte record — all 256 cells share these single values: +/// byte walkMask, wetMask; sbyte sourceZ; sbyte walkZ_N..NW (8); sbyte swimZ_N..NW (8) +/// // Full (Kind == 0) body: /// u8 HasStrata 0 = single-Z chunk (no strata trailer); 1 = strata trailer follows /// u8 HasSwimLayer 0 = no shore cells (no swim trailer); 1 = swim trailer follows /// byte WalkMask[256] @@ -68,7 +72,7 @@ namespace Server.Engines.Pathing.Cache; internal static class StepCacheFile { public const uint Magic = 0x42575300; // 'SWB\0' - public const uint FormatVersion = 4; + public const uint FormatVersion = 5; /// /// Lowest format version this binary can load. Files below this version are treated as @@ -79,9 +83,16 @@ internal static class StepCacheFile /// static-over-land surfaces (sewer/dungeon walkways, bridges, upper building floors), /// producing ~98% source-Z fallthroughs on those routes. The on-disk layout is /// unchanged; only the strata population differs, so the bump exists purely to force a - /// one-time re-bake of stale v3 files on first boot under the new binary. + /// one-time re-bake of stale v3 files on first boot under the new binary. Bumped to 5 for + /// uniform-chunk elision: each record now begins with a Kind byte (0 = Full, 2 = Uniform); + /// a fully-uniform chunk (no strata, no swim layer, all 19 base arrays constant) stores + /// one cell's worth of data (~28 bytes) instead of the full record. /// - public const uint MinSupportedVersion = 4; + public const uint MinSupportedVersion = 5; + + // Per-chunk record discriminator (first byte after BuiltMultisVersion). 1 is reserved. + private const byte KindFull = 0; + private const byte KindUniform = 2; private const int HeaderSize = sizeof(uint) // Magic @@ -100,7 +111,7 @@ internal static class StepCacheFile /// Fixed-size portion of a chunk record (everything except the optional strata + swim trailers). private const int BytesPerChunkBase = sizeof(ushort) + sizeof(ushort) + sizeof(uint) - + sizeof(byte) + sizeof(byte) // HasStrata + HasSwimLayer + + sizeof(byte) + sizeof(byte) + sizeof(byte) // Kind + HasStrata + HasSwimLayer + StepChunk.CellsPerChunk // WalkMask + StepChunk.CellsPerChunk // WetMask + StepChunk.CellsPerChunk // SourceZ @@ -367,6 +378,35 @@ internal static class StepCacheFile w.Write((ushort)chunkY); w.Write((uint)chunk.BuiltMultisVersion); + // Kind: 0 = Full, 2 = Uniform. A uniform chunk (no strata, no swim layer, all 19 base + // arrays constant) stores one cell's worth of data (~28-byte record total). + if (chunk.IsUniform()) + { + w.Write(KindUniform); + w.Write(chunk.WalkMask[0]); + w.Write(chunk.WetMask[0]); + w.Write((byte)chunk.SourceZ[0]); + w.Write((byte)chunk.WalkZN[0]); + w.Write((byte)chunk.WalkZNE[0]); + w.Write((byte)chunk.WalkZE[0]); + w.Write((byte)chunk.WalkZSE[0]); + w.Write((byte)chunk.WalkZS[0]); + w.Write((byte)chunk.WalkZSW[0]); + w.Write((byte)chunk.WalkZW[0]); + w.Write((byte)chunk.WalkZNW[0]); + w.Write((byte)chunk.SwimZN[0]); + w.Write((byte)chunk.SwimZNE[0]); + w.Write((byte)chunk.SwimZE[0]); + w.Write((byte)chunk.SwimZSE[0]); + w.Write((byte)chunk.SwimZS[0]); + w.Write((byte)chunk.SwimZSW[0]); + w.Write((byte)chunk.SwimZW[0]); + w.Write((byte)chunk.SwimZNW[0]); + return; + } + + w.Write(KindFull); // Full + var strataOffsetByCell = chunk.GetStrataOffsetByCellForSerialization(); var strataData = chunk.GetStrataDataForSerialization(); var hasStrata = strataOffsetByCell != null; @@ -433,11 +473,37 @@ internal static class StepCacheFile r.ReadUShort(); r.ReadUShort(); var multisVersion = (int)r.ReadUInt(); - var hasStrata = r.ReadByte() != 0; - var hasSwimLayer = r.ReadByte() != 0; + var kind = r.ReadByte(); var chunk = new StepChunk { BuiltMultisVersion = multisVersion }; + if (kind == KindUniform) // Uniform — one cell's worth of the 19 base arrays, fill all 256 cells. + { + Array.Fill(chunk.WalkMask, r.ReadByte()); + Array.Fill(chunk.WetMask, r.ReadByte()); + Array.Fill(chunk.SourceZ, (sbyte)r.ReadByte()); + Array.Fill(chunk.WalkZN, (sbyte)r.ReadByte()); + Array.Fill(chunk.WalkZNE, (sbyte)r.ReadByte()); + Array.Fill(chunk.WalkZE, (sbyte)r.ReadByte()); + Array.Fill(chunk.WalkZSE, (sbyte)r.ReadByte()); + Array.Fill(chunk.WalkZS, (sbyte)r.ReadByte()); + Array.Fill(chunk.WalkZSW, (sbyte)r.ReadByte()); + Array.Fill(chunk.WalkZW, (sbyte)r.ReadByte()); + Array.Fill(chunk.WalkZNW, (sbyte)r.ReadByte()); + Array.Fill(chunk.SwimZN, (sbyte)r.ReadByte()); + Array.Fill(chunk.SwimZNE, (sbyte)r.ReadByte()); + Array.Fill(chunk.SwimZE, (sbyte)r.ReadByte()); + Array.Fill(chunk.SwimZSE, (sbyte)r.ReadByte()); + Array.Fill(chunk.SwimZS, (sbyte)r.ReadByte()); + Array.Fill(chunk.SwimZSW, (sbyte)r.ReadByte()); + Array.Fill(chunk.SwimZW, (sbyte)r.ReadByte()); + Array.Fill(chunk.SwimZNW, (sbyte)r.ReadByte()); + return chunk; + } + + var hasStrata = r.ReadByte() != 0; + var hasSwimLayer = r.ReadByte() != 0; + r.Read(chunk.WalkMask); r.Read(chunk.WetMask); ReadSBytes(r, chunk.SourceZ); diff --git a/Projects/UOContent/Engines/Pathing/Cache/StepChunk.cs b/Projects/UOContent/Engines/Pathing/Cache/StepChunk.cs index c39039369..6d18b4032 100644 --- a/Projects/UOContent/Engines/Pathing/Cache/StepChunk.cs +++ b/Projects/UOContent/Engines/Pathing/Cache/StepChunk.cs @@ -174,4 +174,29 @@ internal sealed class StepChunk /// Serialization hook: returns the raw data array (or null if no strata). internal byte[] GetStrataDataForSerialization() => _strataData; + + /// + /// True when every cell shares one value across WalkMask, WetMask, SourceZ, and all 16 + /// directional-Z arrays, and the chunk has neither multi-Z strata nor a swim layer. Such a + /// chunk serializes to a ~28-byte uniform record (StepCacheFile v5) instead of the full + /// record. Chunks with a swim layer (shore cells) are never uniform — their per-cell swim + /// data must be preserved via the Full record. + /// + internal bool IsUniform() + { + if (_strataOffsetByCell != null || HasSwimLayer) + { + return false; + } + return AllSame(WalkMask) && AllSame(WetMask) && AllSame(SourceZ) + && AllSame(WalkZN) && AllSame(WalkZNE) && AllSame(WalkZE) && AllSame(WalkZSE) + && AllSame(WalkZS) && AllSame(WalkZSW) && AllSame(WalkZW) && AllSame(WalkZNW) + && AllSame(SwimZN) && AllSame(SwimZNE) && AllSame(SwimZE) && AllSame(SwimZSE) + && AllSame(SwimZS) && AllSame(SwimZSW) && AllSame(SwimZW) && AllSame(SwimZNW); + } + + // "All 256 cells equal" via SIMD-accelerated ContainsAnyExcept (skip cell 0, the reference). + private static bool AllSame(byte[] a) => a.Length < 2 || !a.AsSpan(1).ContainsAnyExcept(a[0]); + + private static bool AllSame(sbyte[] a) => a.Length < 2 || !a.AsSpan(1).ContainsAnyExcept(a[0]); } diff --git a/dev-docs/pathfinding.md b/dev-docs/pathfinding.md index 225cf4c4d..64b331f14 100644 --- a/dev-docs/pathfinding.md +++ b/dev-docs/pathfinding.md @@ -196,7 +196,6 @@ exists to prove this (ratio ≈ 1.0 vs the vendored FastAStar baseline). because the cache's `SourceZ` is computed under default-walker rules, so swim creatures fall through to the slow path. Baking swim-aware source Z (or a swim stratum) would let them hit the cache. Independent of the size-reduction work below. - ### `.swb` size reduction (the ~565 MB → tens of MB roadmap) The v2 format stores every 16×16 chunk as a flat ~5,393-byte record, uncompressed, with no @@ -226,3 +225,15 @@ whole-file) and bounded RAM (only touched chunks materialize, LRU-capped): all-zero Z residuals?) to size the #1/#2 win before writing any format code. Each technique is a clean v3 format bump; `MinSupportedVersion` already silently rejects + overwrites older files. Validate size **and** read-latency vs the corpus in the benchmark repo after each. + +**Measured headroom (Trammel).** Two measurements, both 2026-06-06: + +- *Pre-swim audit (v2 spike, indicative):* directional-Z is **95.2% zero-residual for WalkZ**, + **38.1% for SwimZ** vs `SourceZ` (→ #2; swim wants its own predictor or leans on #3). The + spike's combined size projection double-counted and is superseded by the calibration below. +- *Calibrated on the real v4/v5 format (`SaveToFile`-measured):* 114,688 chunks; **62.7% fully + uniform** (swim-aware → #1 elides these), **8.9% carry a swim layer** and **1.8% strata** + (both stay Full). **#1 alone: 592.2 MB → 231.9 MB (−61%), actual on-disk.** The residual is + ~150 MB of non-uniform land Z-blocks (→ #2 predictive-Z) + ~81 MB of swim-layer trailers + (→ #2/#3). Confirms build order **#1 → #2 → #3**, with #1 the dominant, lowest-risk, + no-algorithm-change win (now shipped as format v5).