diff --git a/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV5Tests.cs b/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV6Tests.cs
similarity index 52%
rename from Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV5Tests.cs
rename to Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV6Tests.cs
index 32ce16d10..30d8630c1 100644
--- a/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV5Tests.cs
+++ b/Projects/UOContent.Tests/Tests/Engines/Pathing/StepCacheFileV6Tests.cs
@@ -5,29 +5,72 @@ using Xunit;
namespace Server.Tests.Pathfinding;
-// v5 = uniform-chunk elision on top of the v4 swim-layer format. A uniform chunk (no strata,
-// no swim layer, all 19 base arrays constant) serializes to ~28 bytes; Full chunks (incl. swim
-// layer + strata) round-trip byte-identically.
+// v6 = predictive-Z residuals on top of the v5 uniform-elision format. Each base directional
+// Z array is stored as a masked residual against the cell's own SourceZ; arrays that match
+// their prediction are omitted entirely (ZArrayMask bit clear) and synthesized at read.
[Collection("Sequential Pathfinding Tests")]
-public class StepCacheFileV5Tests
+public class StepCacheFileV6Tests
{
- private static StepChunk UniformChunk(byte walk = 0xC1, byte wet = 0x00, sbyte z = 10, int multis = 7)
+ [Theory]
+ [InlineData((sbyte)0, (sbyte)0)]
+ [InlineData((sbyte)10, (sbyte)10)]
+ [InlineData((sbyte)0, (sbyte)10)]
+ [InlineData((sbyte)10, (sbyte)0)]
+ [InlineData((sbyte)-20, (sbyte)15)]
+ [InlineData(sbyte.MinValue, sbyte.MaxValue)]
+ [InlineData(sbyte.MaxValue, sbyte.MinValue)]
+ [InlineData(sbyte.MinValue, (sbyte)1)]
+ [InlineData((sbyte)127, (sbyte)-1)]
+ public void Residual_RoundTrips_Losslessly_ForAllInputs(sbyte z, sbyte predict)
+ {
+ var residual = StepCacheFile.EncodeResidual(z, predict);
+ Assert.Equal(z, StepCacheFile.DecodeZ(predict, residual));
+ }
+
+ [Theory]
+ [InlineData((byte)0b0000_0001, 0, (sbyte)42, (sbyte)42)] // bit set -> sourceZ
+ [InlineData((byte)0b0000_0000, 0, (sbyte)42, (sbyte)0)] // bit clear -> 0
+ [InlineData((byte)0b1000_0000, 7, (sbyte)-13, (sbyte)-13)]
+ [InlineData((byte)0b0111_1111, 7, (sbyte)-13, (sbyte)0)]
+ public void Predict_UsesSourceZWhenBitSet_ZeroOtherwise(byte maskByte, int bit, sbyte sourceZ, sbyte expected)
+ {
+ Assert.Equal(expected, StepCacheFile.Predict(maskByte, bit, sourceZ));
+ }
+
+ // ---- builders ----
+
+ // A FULL chunk (not uniform: masks/SourceZ vary per cell) whose every directional-Z equals
+ // its masked prediction => all 16 base Z arrays must elide. Doubles as the coastline case:
+ // per-cell partial walkability with SourceZ != 0, flat where walkable, 0 where not.
+ private static StepChunk FlatFullChunk(int multis = 3, sbyte baseZ = 10)
{
var c = new StepChunk { BuiltMultisVersion = multis };
- Array.Fill(c.WalkMask, walk);
- Array.Fill(c.WetMask, wet);
- Array.Fill(c.SourceZ, z);
- foreach (var arr in new[]
- {
- c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW,
- c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW
- })
+ for (var i = 0; i < StepChunk.CellsPerChunk; i++)
{
- Array.Fill(arr, z);
+ c.WalkMask[i] = (byte)(i & 0xFF);
+ c.WetMask[i] = (byte)((~i) & 0xFF);
+ c.SourceZ[i] = (sbyte)(baseZ + (i % 7) - 3); // varies, mostly != 0
}
+ SetFlatDirectional(c);
return c;
}
+ // Sets every directional-Z to its masked prediction (walkable/wet -> SourceZ, else 0),
+ // i.e. perfectly flat terrain. Such arrays all elide under v6.
+ private static void SetFlatDirectional(StepChunk c)
+ {
+ var walk = new[] { c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW };
+ var swim = new[] { c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW };
+ for (var i = 0; i < StepChunk.CellsPerChunk; i++)
+ {
+ for (var b = 0; b < 8; b++)
+ {
+ walk[b][i] = (sbyte)((c.WalkMask[i] >> b & 1) != 0 ? c.SourceZ[i] : 0);
+ swim[b][i] = (sbyte)((c.WetMask[i] >> b & 1) != 0 ? c.SourceZ[i] : 0);
+ }
+ }
+ }
+
private static StepChunk VariedChunk(int multis = 3)
{
var c = new StepChunk { BuiltMultisVersion = multis };
@@ -56,6 +99,35 @@ public class StepCacheFileV5Tests
return c;
}
+ // ---- round-trip plumbing ----
+
+ private static string Write1(StepChunk c, int cx, int cy)
+ {
+ var path = Path.Combine(Path.GetTempPath(), $"swbv6_{Guid.NewGuid():N}.swb");
+ var emitted = false;
+ StepCacheFile.Write(path, 1u, 1u, (out int ox, out int oy, out StepChunk oc) =>
+ {
+ if (emitted) { ox = oy = 0; oc = null!; return false; }
+ emitted = true; ox = cx; oy = cy; oc = c; return true;
+ });
+ return path;
+ }
+
+ private static StepChunk RoundTrip(StepChunk src, int cx, int cy, out long fileLen)
+ {
+ var path = Write1(src, cx, cy);
+ try
+ {
+ fileLen = new FileInfo(path).Length;
+ using var reader = StepCacheFile.OpenForLazy(path);
+ Assert.NotNull(reader);
+ var rt = reader!.TryReadChunk(cx, cy);
+ Assert.NotNull(rt);
+ return rt!;
+ }
+ finally { File.Delete(path); }
+ }
+
private static void AssertChunksEqual(StepChunk a, StepChunk b)
{
Assert.Equal(a.BuiltMultisVersion, b.BuiltMultisVersion);
@@ -88,61 +160,42 @@ public class StepCacheFileV5Tests
}
}
- private static string Write1(StepChunk c, int cx, int cy)
- {
- var path = Path.Combine(Path.GetTempPath(), $"swbv5_{Guid.NewGuid():N}.swb");
- var emitted = false;
- StepCacheFile.Write(path, 1u, 1u, (out int ox, out int oy, out StepChunk oc) =>
- {
- if (emitted) { ox = oy = 0; oc = null!; return false; }
- emitted = true; ox = cx; oy = cy; oc = c; return true;
- });
- return path;
- }
-
- private static StepChunk RoundTrip(StepChunk src, int cx, int cy, out long fileLen)
- {
- var path = Write1(src, cx, cy);
- try
- {
- fileLen = new FileInfo(path).Length;
- using var reader = StepCacheFile.OpenForLazy(path);
- Assert.NotNull(reader);
- var rt = reader!.TryReadChunk(cx, cy);
- Assert.NotNull(rt);
- return rt!;
- }
- finally { File.Delete(path); }
- }
+ // ---- transform tests (Task 2) ----
[Fact]
- public void IsUniform_TrueForAllIdentical_FalseForVariedStrataOrSwim()
+ public void FlatFull_AllArraysElide_RoundTripsAndIsCompact()
{
- Assert.True(UniformChunk().IsUniform());
-
- var varied = UniformChunk();
- varied.WalkZE[42] = 99;
- Assert.False(varied.IsUniform());
-
- var strata = UniformChunk();
- var offsets = new ushort[StepChunk.CellsPerChunk];
- Array.Fill(offsets, StepChunk.NoStrata);
- offsets[0] = 0;
- strata.SetStrata(offsets, new byte[] { 0 });
- Assert.False(strata.IsUniform());
-
- var swim = UniformChunk();
- swim.AllocateSwimLayer(); // a uniform-looking base but with a swim layer is NOT uniform
- Assert.False(swim.IsUniform());
- }
-
- [Fact]
- public void Uniform_RoundTrips_Identically_AndIsCompact()
- {
- var src = UniformChunk(walk: 0xC1, wet: 0x00, z: 12, multis: 9);
+ var src = FlatFullChunk();
var rt = RoundTrip(src, 5, 6, out var fileLen);
- Assert.True(fileLen < 200, $"uniform .swb too large: {fileLen}");
AssertChunksEqual(src, rt);
+ // Full record with all 16 Z arrays elided: header(48) + ~783-byte record + index(20).
+ // A v5 full record alone is > 5 KB, so a sub-1100-byte file proves elision fired.
+ Assert.True(fileLen < 1100, $"expected all base Z arrays to elide; file was {fileLen} bytes");
+ }
+
+ [Fact]
+ public void SlopedSubset_OnlyVaryingArraysPresent_RoundTrips()
+ {
+ var flat = FlatFullChunk();
+ var flatPath = Write1(flat, 1, 1);
+ long flatLen;
+ try { flatLen = new FileInfo(flatPath).Length; } finally { File.Delete(flatPath); }
+
+ // Bump WalkZN by +1 on cells walkable to the N (slope in one direction only) -> exactly
+ // one base Z array (WalkZN, bit 0) becomes present; the other 15 still elide.
+ var sloped = FlatFullChunk();
+ for (var i = 0; i < StepChunk.CellsPerChunk; i++)
+ {
+ if ((sloped.WalkMask[i] & 1) != 0)
+ {
+ sloped.WalkZN[i] = (sbyte)(sloped.WalkZN[i] + 1);
+ }
+ }
+
+ var rt = RoundTrip(sloped, 2, 3, out var slopedLen);
+ AssertChunksEqual(sloped, rt);
+ Assert.True(slopedLen > flatLen, "one present array should grow the record vs all-flat");
+ Assert.True(slopedLen <= flatLen + StepChunk.CellsPerChunk, "only one 256-byte residual array should be added");
}
[Fact]
@@ -152,6 +205,20 @@ public class StepCacheFileV5Tests
AssertChunksEqual(src, RoundTrip(src, 1, 2, out _));
}
+ // ---- shape coverage (Task 3) ----
+
+ [Fact]
+ public void Coastline_NonzeroSourceZ_PartialWalkability_AllElide()
+ {
+ // FlatFullChunk already models a coastline: per-cell partial walk/wet masks, SourceZ != 0,
+ // flat where walkable and 0 (baker default) where not. A plain SourceZ residual would emit
+ // -SourceZ on every unwalkable direction; the masked predictor must drive ALL arrays to elide.
+ var src = FlatFullChunk(multis: 2, baseZ: 25);
+ var rt = RoundTrip(src, 7, 7, out var fileLen);
+ AssertChunksEqual(src, rt);
+ Assert.True(fileLen < 1100, $"masked predictor should elide every array on flat coastline; file was {fileLen} bytes");
+ }
+
[Fact]
public void SwimLayer_Full_RoundTrips_Identically()
{
@@ -178,24 +245,10 @@ public class StepCacheFileV5Tests
Assert.True(rt.StrataData.SequenceEqual(src.StrataData));
}
- [Fact]
- public void OlderVersion_IsRejected()
- {
- var path = Write1(UniformChunk(), 0, 0);
- try
- {
- var bytes = File.ReadAllBytes(path);
- bytes[4] = 4; bytes[5] = 0; bytes[6] = 0; bytes[7] = 0; // version 4 < MinSupportedVersion 5
- File.WriteAllBytes(path, bytes);
- Assert.Null(StepCacheFile.OpenForLazy(path));
- }
- finally { File.Delete(path); }
- }
-
[Fact]
public void SwimAndStrata_Full_RoundTrips_Identically()
{
- // Exercises the combined trailer ordering: swim-layer trailer THEN strata trailer.
+ // Combined trailer ordering: swim-layer trailer THEN strata trailer, after the residual blocks.
var src = SwimChunk(multis: 11);
var offsets = new ushort[StepChunk.CellsPerChunk];
Array.Fill(offsets, StepChunk.NoStrata);
@@ -210,4 +263,18 @@ public class StepCacheFileV5Tests
AssertChunksEqual(src, rt);
Assert.True(rt.StrataData.SequenceEqual(src.StrataData));
}
+
+ [Fact]
+ public void OlderVersion_IsRejected()
+ {
+ var path = Write1(FlatFullChunk(), 0, 0);
+ try
+ {
+ var bytes = File.ReadAllBytes(path);
+ bytes[4] = 5; bytes[5] = 0; bytes[6] = 0; bytes[7] = 0; // version 5 < MinSupportedVersion 6
+ File.WriteAllBytes(path, bytes);
+ Assert.Null(StepCacheFile.OpenForLazy(path));
+ }
+ finally { File.Delete(path); }
+ }
}
diff --git a/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs b/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs
index 4e1df83bd..c34412604 100644
--- a/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs
+++ b/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs
@@ -14,11 +14,11 @@ namespace Server.Engines.Pathing.Cache;
/// only when the cache asks for them. RAM stays bounded by MaxResidentChunks regardless
/// of file size.
///
-/// File layout v5 (little-endian, BufferWriter / BufferReader convention):
+/// File layout v6 (little-endian, BufferWriter / BufferReader convention):
///
/// Header (48 bytes):
/// u32 Magic = 0x42575300 ('SWB\0')
-/// u32 Version = current FormatVersion (5)
+/// u32 Version = current FormatVersion (6)
/// u32 MapId
/// u64 Fingerprint XxHash3 over (1) LandTable + ItemTable flags AND (2) the
/// on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul.
@@ -40,11 +40,16 @@ namespace Server.Engines.Pathing.Cache;
/// // Full (Kind == 0) body:
/// u8 HasStrata 0 = single-Z chunk (no strata trailer); 1 = strata trailer follows
/// u8 HasSwimLayer 0 = no shore cells (no swim trailer); 1 = swim trailer follows
+/// u16 ZArrayMask bit d set => base directional Z array d is present below as a
+/// residual[256] block; cleared => array equals its prediction and is
+/// omitted (synthesized at read). bits 0-7 = WalkZ N..NW (predicted via
+/// WalkMask), bits 8-15 = SwimZ N..NW (predicted via WetMask).
/// byte WalkMask[256]
/// byte WetMask[256]
/// sbyte SourceZ[256]
-/// sbyte WalkZN[256]..WalkZNW[256] (8 arrays in N,NE,E,SE,S,SW,W,NW order)
-/// sbyte SwimZN[256]..SwimZNW[256] (8 arrays in same order; baked at WALK source-Z)
+/// // For each d in 0..15 with ZArrayMask bit d set, in N,NE,E,SE,S,SW,W,NW order
+/// // (walk arrays first, then swim):
+/// sbyte residual_d[256] reconstruct: Z_d[c] = (mask bit set ? SourceZ[c] : 0) + residual_d[c]
/// // Swim layer trailer — only when HasSwimLayer == 1 (chunks containing shore cells):
/// sbyte SwimSourceZ[256] (NoSwimLayerCell sentinel = sbyte.MinValue)
/// byte SwimMask[256] (per-cell swim mask baked at SwimSourceZ)
@@ -62,8 +67,10 @@ namespace Server.Engines.Pathing.Cache;
/// Index trailer (20 × ChunkCount bytes):
/// For each chunk: { u64 chunkKey, u64 fileOffset, u32 recordLength }
///
-/// Per-chunk fixed portion: ~5,393 bytes. Strata trailer: 516 + N × ~30 bytes for a
-/// chunk with N multi-Z cells averaging ~2 strata each. LRU bookkeeping
+/// Per-chunk fixed portion (Kind + flags + ZArrayMask + WalkMask + WetMask + SourceZ):
+/// ~783 bytes; each present base Z array adds 256 bytes (0..16 present, so up to ~4 KB).
+/// A fully-flat Full chunk stores no residual blocks. Strata trailer: 516 + N × ~30 bytes
+/// for a chunk with N multi-Z cells averaging ~2 strata each. LRU bookkeeping
/// (LastTouchedTicks) is intentionally not persisted.
///
/// Files with version < are silently rejected
@@ -72,7 +79,7 @@ namespace Server.Engines.Pathing.Cache;
internal static class StepCacheFile
{
public const uint Magic = 0x42575300; // 'SWB\0'
- public const uint FormatVersion = 5;
+ public const uint FormatVersion = 6;
///
/// Lowest format version this binary can load. Files below this version are treated as
@@ -86,9 +93,12 @@ internal static class StepCacheFile
/// one-time re-bake of stale v3 files on first boot under the new binary. Bumped to 5 for
/// uniform-chunk elision: each record now begins with a Kind byte (0 = Full, 2 = Uniform);
/// a fully-uniform chunk (no strata, no swim layer, all 19 base arrays constant) stores
- /// one cell's worth of data (~28 bytes) instead of the full record.
+ /// one cell's worth of data (~28 bytes) instead of the full record. Bumped to 6 for
+ /// predictive-Z residuals: each base directional Z array is stored as a masked residual
+ /// against SourceZ (a ZArrayMask u16 flags which arrays are present); arrays matching their
+ /// prediction are omitted and synthesized at read. v5 files are rejected and re-baked once.
///
- public const uint MinSupportedVersion = 5;
+ public const uint MinSupportedVersion = 6;
// Per-chunk record discriminator (first byte after BuiltMultisVersion). 1 is reserved.
private const byte KindFull = 0;
@@ -112,6 +122,7 @@ internal static class StepCacheFile
private const int BytesPerChunkBase =
sizeof(ushort) + sizeof(ushort) + sizeof(uint)
+ sizeof(byte) + sizeof(byte) + sizeof(byte) // Kind + HasStrata + HasSwimLayer
+ + sizeof(ushort) // ZArrayMask
+ StepChunk.CellsPerChunk // WalkMask
+ StepChunk.CellsPerChunk // WetMask
+ StepChunk.CellsPerChunk // SourceZ
@@ -372,6 +383,35 @@ internal static class StepCacheFile
private static ulong PackChunkKey(int chunkX, int chunkY) => ((ulong)(uint)chunkX << 32) | (uint)chunkY;
+ ///
+ /// Predicted directional-Z for one cell/direction: the cell's own SourceZ when the
+ /// direction is walkable/wet (mask bit set), else 0 — matching the baker, which leaves
+ /// non-walkable directional slots at their zero-initialized default
+ /// (StepProbe.ComputeMaskAt clears walkZs/swimZs and writes only on a successful step).
+ ///
+ internal static sbyte Predict(byte dirMaskByte, int bit, sbyte sourceZ) =>
+ (dirMaskByte >> bit & 1) != 0 ? sourceZ : (sbyte)0;
+
+ ///
+ /// Residual of an absolute directional-Z against its prediction. Unchecked two's-complement
+ /// so the transform is byte-exact for ALL sbyte inputs (no value-range constraint).
+ ///
+ internal static sbyte EncodeResidual(sbyte z, sbyte predict) => unchecked((sbyte)(z - predict));
+
+ /// Inverse of : absolute directional-Z = predict + residual.
+ internal static sbyte DecodeZ(sbyte predict, sbyte residual) => unchecked((sbyte)(predict + residual));
+
+ ///
+ /// The 16 base directional-Z arrays in canonical order: walk N..NW (indices 0-7),
+ /// then swim N..NW (8-15). Index d uses WalkMask (d < 8) or WetMask (d >= 8)
+ /// with direction bit (d & 7). Allocates a 16-slot reference array (bake/read time only).
+ ///
+ private static sbyte[][] GetBaseZArrays(StepChunk c) => new[]
+ {
+ c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW,
+ c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW,
+ };
+
private static void WriteChunk(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
{
w.Write((ushort)chunkX);
@@ -414,27 +454,47 @@ internal static class StepCacheFile
w.Write((byte)(hasStrata ? 1 : 0));
w.Write((byte)(hasSwimLayer ? 1 : 0));
+ // Predictive-Z: each base directional Z array is stored as a masked residual against
+ // SourceZ. Bit d of ZArrayMask is set only when array d differs from its prediction
+ // somewhere; cleared arrays are omitted and rebuilt from mask+SourceZ at read.
+ var zArrays = GetBaseZArrays(chunk);
+ ushort zArrayMask = 0;
+ for (var d = 0; d < 16; d++)
+ {
+ var z = zArrays[d];
+ var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
+ var bit = d & 7;
+ for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
+ {
+ if (z[cell] != Predict(dirMask[cell], bit, chunk.SourceZ[cell]))
+ {
+ zArrayMask |= (ushort)(1 << d);
+ break;
+ }
+ }
+ }
+ w.Write(zArrayMask);
+
w.Write(chunk.WalkMask);
w.Write(chunk.WetMask);
WriteSBytes(w, chunk.SourceZ);
- WriteSBytes(w, chunk.WalkZN);
- WriteSBytes(w, chunk.WalkZNE);
- WriteSBytes(w, chunk.WalkZE);
- WriteSBytes(w, chunk.WalkZSE);
- WriteSBytes(w, chunk.WalkZS);
- WriteSBytes(w, chunk.WalkZSW);
- WriteSBytes(w, chunk.WalkZW);
- WriteSBytes(w, chunk.WalkZNW);
-
- WriteSBytes(w, chunk.SwimZN);
- WriteSBytes(w, chunk.SwimZNE);
- WriteSBytes(w, chunk.SwimZE);
- WriteSBytes(w, chunk.SwimZSE);
- WriteSBytes(w, chunk.SwimZS);
- WriteSBytes(w, chunk.SwimZSW);
- WriteSBytes(w, chunk.SwimZW);
- WriteSBytes(w, chunk.SwimZNW);
+ Span residual = stackalloc sbyte[StepChunk.CellsPerChunk];
+ for (var d = 0; d < 16; d++)
+ {
+ if ((zArrayMask >> d & 1) == 0)
+ {
+ continue;
+ }
+ var z = zArrays[d];
+ var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
+ var bit = d & 7;
+ for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
+ {
+ residual[cell] = EncodeResidual(z[cell], Predict(dirMask[cell], bit, chunk.SourceZ[cell]));
+ }
+ w.Write(MemoryMarshal.Cast(residual));
+ }
if (hasSwimLayer)
{
@@ -503,28 +563,37 @@ internal static class StepCacheFile
var hasStrata = r.ReadByte() != 0;
var hasSwimLayer = r.ReadByte() != 0;
+ var zArrayMask = r.ReadUShort();
r.Read(chunk.WalkMask);
r.Read(chunk.WetMask);
ReadSBytes(r, chunk.SourceZ);
- ReadSBytes(r, chunk.WalkZN);
- ReadSBytes(r, chunk.WalkZNE);
- ReadSBytes(r, chunk.WalkZE);
- ReadSBytes(r, chunk.WalkZSE);
- ReadSBytes(r, chunk.WalkZS);
- ReadSBytes(r, chunk.WalkZSW);
- ReadSBytes(r, chunk.WalkZW);
- ReadSBytes(r, chunk.WalkZNW);
-
- ReadSBytes(r, chunk.SwimZN);
- ReadSBytes(r, chunk.SwimZNE);
- ReadSBytes(r, chunk.SwimZE);
- ReadSBytes(r, chunk.SwimZSE);
- ReadSBytes(r, chunk.SwimZS);
- ReadSBytes(r, chunk.SwimZSW);
- ReadSBytes(r, chunk.SwimZW);
- ReadSBytes(r, chunk.SwimZNW);
+ // Predictive-Z reconstruction: present arrays carry residuals (z = predict + residual);
+ // absent arrays are synthesized from mask+SourceZ (z = predict, residual implicitly 0).
+ var zArrays = GetBaseZArrays(chunk);
+ Span residual = stackalloc sbyte[StepChunk.CellsPerChunk];
+ for (var d = 0; d < 16; d++)
+ {
+ var z = zArrays[d];
+ var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
+ var bit = d & 7;
+ if ((zArrayMask >> d & 1) != 0)
+ {
+ r.Read(MemoryMarshal.Cast(residual));
+ for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
+ {
+ z[cell] = DecodeZ(Predict(dirMask[cell], bit, chunk.SourceZ[cell]), residual[cell]);
+ }
+ }
+ else
+ {
+ for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
+ {
+ z[cell] = Predict(dirMask[cell], bit, chunk.SourceZ[cell]);
+ }
+ }
+ }
if (hasSwimLayer)
{
diff --git a/dev-docs/pathfinding.md b/dev-docs/pathfinding.md
index 64b331f14..0530dceb8 100644
--- a/dev-docs/pathfinding.md
+++ b/dev-docs/pathfinding.md
@@ -208,13 +208,19 @@ whole-file) and bounded RAM (only touched chunks materialize, LRU-capped):
mask + Z is stored as a few bytes (flag + mask + Z) instead of 5,393. Pairs with a two-level
**sector index** so a uniform super-sector (open ocean) collapses to a single entry — UO's
16-tile sectors nest cleanly inside.
-2. **Predictive (lossless) Z residuals** (kills the 76% that is the 16 directional Z arrays).
- Predict each `WalkZ_dir`/`SwimZ_dir` from the cell's **own `SourceZ`** and store only the
- residual: 0 on flat ground (→ compresses to nothing → effectively just the walk *bit*),
- ±1..3 on slopes/stairs, larger or strata for bridges/multi-Z. Reconstruct
- `WalkZ_dir = SourceZ + residual` at load → byte-identical in-memory `StepChunk`, so **no
- runtime recompute and no risk of diverging from `MovementImpl`** (the bugs the cache exists
- to avoid). Predict from *self* (not a neighbor) to keep chunk independence.
+2. **Predictive (lossless) Z residuals** — *shipped as format v6.* Kills the bulk of the 16
+ base directional Z arrays in Full chunks. Each array is stored as a **masked residual**
+ against the cell's **own `SourceZ`**: predict `mask bit ? SourceZ : 0` (the mask term matches
+ the baker, which leaves non-walkable directional slots at `0`), residual `= Z − predict` via
+ unchecked two's-complement (byte-exact for all inputs). A new u16 `ZArrayMask` flags which of
+ the 16 arrays differ from their prediction; an array that matches everywhere (flat terrain —
+ including partial-walkability coastlines with nonzero `SourceZ`) is **omitted entirely** and
+ synthesized from `mask + SourceZ` at read. Reconstruct `Z = predict + residual` → byte-identical
+ in-memory `StepChunk`, so **no runtime recompute and no risk of diverging from `MovementImpl`**.
+ Self-prediction (not a neighbor) keeps chunk independence. The residual *subtraction* itself
+ saves ~0 bytes (a residual is still 1 byte/cell); the win is the per-array elision, and leaving
+ non-elided arrays in residual form makes #3 a pure codec add (no further transform/format bump).
+ Swim-layer + strata trailers stay absolute, deferred to #3.
3. **Per-chunk block compression + index compaction.** zstd/deflate each chunk record
independently (index already carries a per-chunk `length`; reader decompresses one chunk on
read). At 256-cell granularity the **index overhead** then dominates (20 B/chunk ×
@@ -236,4 +242,12 @@ Validate size **and** read-latency vs the corpus in the benchmark repo after eac
(both stay Full). **#1 alone: 592.2 MB → 231.9 MB (−61%), actual on-disk.** The residual is
~150 MB of non-uniform land Z-blocks (→ #2 predictive-Z) + ~81 MB of swim-layer trailers
(→ #2/#3). Confirms build order **#1 → #2 → #3**, with #1 the dominant, lowest-risk,
- no-algorithm-change win (now shipped as format v5).
+ no-algorithm-change win (shipped as format v5).
+- *Calibrated v6 (`BakeMap`-measured, full Trammel, `MaxResidentChunks` raised above 114,688):*
+ **#2 predictive-Z: 231.9 MB → 124.7 MB (−46%, −107 MB; −78% vs the original 565 MB).** Of the
+ 42,775 Full chunks (71,913 are Uniform), **walk directional-Z arrays elide 47.7%** and **swim
+ arrays elide 87.6%** (per-array, not per-cell — one slope cell keeps a 256-byte array, which is
+ why the per-array rate trails the 95.2% per-cell zero-residual). Remaining v6 bytes are dominated
+ by present walk-residual blocks (~46 MB, mostly ±1..3 + zeros → highly compressible), the
+ per-Full-chunk mask/SourceZ base (~33 MB), and absolute swim-layer trailers (~26 MB) — all prime
+ targets for #3 per-chunk compression.