feat(pathfinding): .swb format v6 predictive-Z residuals (#2469)
## Summary Phase #2 of the `.swb` step-cache size-reduction roadmap (after #2465, v5 uniform elision). Stores the 16 base directional Z arrays as masked residuals against each cell's own SourceZ and omits any array that matches its prediction. Lossless, byte-identical reconstruction. Trammel: 231.9 MB → 124.7 MB (−46%). ## Details - Predictor: `predict = mask bit ? SourceZ : 0` (matches the baker's 0 on unwalkable directions); residual `Z − predict` via unchecked two's-complement (byte-exact for all inputs); reconstruct `Z = predict + residual`. - A `u16 ZArrayMask` flags which of the 16 base arrays differ from prediction; matching arrays are omitted and synthesized from mask + SourceZ at read. - Serializer-layer only: StepChunk, the cache, the algorithm, and the baker are unchanged. - Format v6; v5 files rejected and re-baked once. ## Tests 21 v6 unit tests; full pathfinding suite green; Release build clean.
This commit is contained in:
Kamron Batman
GitHub
parent
8e5e4f72c8
commit
94537f83f8
3 changed files with 280 additions and 130 deletions
|
|
@ -5,29 +5,72 @@ using Xunit;
|
|||
|
||||
namespace Server.Tests.Pathfinding;
|
||||
|
||||
// v5 = uniform-chunk elision on top of the v4 swim-layer format. A uniform chunk (no strata,
|
||||
// no swim layer, all 19 base arrays constant) serializes to ~28 bytes; Full chunks (incl. swim
|
||||
// layer + strata) round-trip byte-identically.
|
||||
// v6 = predictive-Z residuals on top of the v5 uniform-elision format. Each base directional
|
||||
// Z array is stored as a masked residual against the cell's own SourceZ; arrays that match
|
||||
// their prediction are omitted entirely (ZArrayMask bit clear) and synthesized at read.
|
||||
[Collection("Sequential Pathfinding Tests")]
|
||||
public class StepCacheFileV5Tests
|
||||
public class StepCacheFileV6Tests
|
||||
{
|
||||
private static StepChunk UniformChunk(byte walk = 0xC1, byte wet = 0x00, sbyte z = 10, int multis = 7)
|
||||
[Theory]
|
||||
[InlineData((sbyte)0, (sbyte)0)]
|
||||
[InlineData((sbyte)10, (sbyte)10)]
|
||||
[InlineData((sbyte)0, (sbyte)10)]
|
||||
[InlineData((sbyte)10, (sbyte)0)]
|
||||
[InlineData((sbyte)-20, (sbyte)15)]
|
||||
[InlineData(sbyte.MinValue, sbyte.MaxValue)]
|
||||
[InlineData(sbyte.MaxValue, sbyte.MinValue)]
|
||||
[InlineData(sbyte.MinValue, (sbyte)1)]
|
||||
[InlineData((sbyte)127, (sbyte)-1)]
|
||||
public void Residual_RoundTrips_Losslessly_ForAllInputs(sbyte z, sbyte predict)
|
||||
{
|
||||
var residual = StepCacheFile.EncodeResidual(z, predict);
|
||||
Assert.Equal(z, StepCacheFile.DecodeZ(predict, residual));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData((byte)0b0000_0001, 0, (sbyte)42, (sbyte)42)] // bit set -> sourceZ
|
||||
[InlineData((byte)0b0000_0000, 0, (sbyte)42, (sbyte)0)] // bit clear -> 0
|
||||
[InlineData((byte)0b1000_0000, 7, (sbyte)-13, (sbyte)-13)]
|
||||
[InlineData((byte)0b0111_1111, 7, (sbyte)-13, (sbyte)0)]
|
||||
public void Predict_UsesSourceZWhenBitSet_ZeroOtherwise(byte maskByte, int bit, sbyte sourceZ, sbyte expected)
|
||||
{
|
||||
Assert.Equal(expected, StepCacheFile.Predict(maskByte, bit, sourceZ));
|
||||
}
|
||||
|
||||
// ---- builders ----
|
||||
|
||||
// A FULL chunk (not uniform: masks/SourceZ vary per cell) whose every directional-Z equals
|
||||
// its masked prediction => all 16 base Z arrays must elide. Doubles as the coastline case:
|
||||
// per-cell partial walkability with SourceZ != 0, flat where walkable, 0 where not.
|
||||
private static StepChunk FlatFullChunk(int multis = 3, sbyte baseZ = 10)
|
||||
{
|
||||
var c = new StepChunk { BuiltMultisVersion = multis };
|
||||
Array.Fill(c.WalkMask, walk);
|
||||
Array.Fill(c.WetMask, wet);
|
||||
Array.Fill(c.SourceZ, z);
|
||||
foreach (var arr in new[]
|
||||
{
|
||||
c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW,
|
||||
c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW
|
||||
})
|
||||
for (var i = 0; i < StepChunk.CellsPerChunk; i++)
|
||||
{
|
||||
Array.Fill(arr, z);
|
||||
c.WalkMask[i] = (byte)(i & 0xFF);
|
||||
c.WetMask[i] = (byte)((~i) & 0xFF);
|
||||
c.SourceZ[i] = (sbyte)(baseZ + (i % 7) - 3); // varies, mostly != 0
|
||||
}
|
||||
SetFlatDirectional(c);
|
||||
return c;
|
||||
}
|
||||
|
||||
// Sets every directional-Z to its masked prediction (walkable/wet -> SourceZ, else 0),
|
||||
// i.e. perfectly flat terrain. Such arrays all elide under v6.
|
||||
private static void SetFlatDirectional(StepChunk c)
|
||||
{
|
||||
var walk = new[] { c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW };
|
||||
var swim = new[] { c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW };
|
||||
for (var i = 0; i < StepChunk.CellsPerChunk; i++)
|
||||
{
|
||||
for (var b = 0; b < 8; b++)
|
||||
{
|
||||
walk[b][i] = (sbyte)((c.WalkMask[i] >> b & 1) != 0 ? c.SourceZ[i] : 0);
|
||||
swim[b][i] = (sbyte)((c.WetMask[i] >> b & 1) != 0 ? c.SourceZ[i] : 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static StepChunk VariedChunk(int multis = 3)
|
||||
{
|
||||
var c = new StepChunk { BuiltMultisVersion = multis };
|
||||
|
|
@ -56,6 +99,35 @@ public class StepCacheFileV5Tests
|
|||
return c;
|
||||
}
|
||||
|
||||
// ---- round-trip plumbing ----
|
||||
|
||||
private static string Write1(StepChunk c, int cx, int cy)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), $"swbv6_{Guid.NewGuid():N}.swb");
|
||||
var emitted = false;
|
||||
StepCacheFile.Write(path, 1u, 1u, (out int ox, out int oy, out StepChunk oc) =>
|
||||
{
|
||||
if (emitted) { ox = oy = 0; oc = null!; return false; }
|
||||
emitted = true; ox = cx; oy = cy; oc = c; return true;
|
||||
});
|
||||
return path;
|
||||
}
|
||||
|
||||
private static StepChunk RoundTrip(StepChunk src, int cx, int cy, out long fileLen)
|
||||
{
|
||||
var path = Write1(src, cx, cy);
|
||||
try
|
||||
{
|
||||
fileLen = new FileInfo(path).Length;
|
||||
using var reader = StepCacheFile.OpenForLazy(path);
|
||||
Assert.NotNull(reader);
|
||||
var rt = reader!.TryReadChunk(cx, cy);
|
||||
Assert.NotNull(rt);
|
||||
return rt!;
|
||||
}
|
||||
finally { File.Delete(path); }
|
||||
}
|
||||
|
||||
private static void AssertChunksEqual(StepChunk a, StepChunk b)
|
||||
{
|
||||
Assert.Equal(a.BuiltMultisVersion, b.BuiltMultisVersion);
|
||||
|
|
@ -88,61 +160,42 @@ public class StepCacheFileV5Tests
|
|||
}
|
||||
}
|
||||
|
||||
private static string Write1(StepChunk c, int cx, int cy)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), $"swbv5_{Guid.NewGuid():N}.swb");
|
||||
var emitted = false;
|
||||
StepCacheFile.Write(path, 1u, 1u, (out int ox, out int oy, out StepChunk oc) =>
|
||||
{
|
||||
if (emitted) { ox = oy = 0; oc = null!; return false; }
|
||||
emitted = true; ox = cx; oy = cy; oc = c; return true;
|
||||
});
|
||||
return path;
|
||||
}
|
||||
|
||||
private static StepChunk RoundTrip(StepChunk src, int cx, int cy, out long fileLen)
|
||||
{
|
||||
var path = Write1(src, cx, cy);
|
||||
try
|
||||
{
|
||||
fileLen = new FileInfo(path).Length;
|
||||
using var reader = StepCacheFile.OpenForLazy(path);
|
||||
Assert.NotNull(reader);
|
||||
var rt = reader!.TryReadChunk(cx, cy);
|
||||
Assert.NotNull(rt);
|
||||
return rt!;
|
||||
}
|
||||
finally { File.Delete(path); }
|
||||
}
|
||||
// ---- transform tests (Task 2) ----
|
||||
|
||||
[Fact]
|
||||
public void IsUniform_TrueForAllIdentical_FalseForVariedStrataOrSwim()
|
||||
public void FlatFull_AllArraysElide_RoundTripsAndIsCompact()
|
||||
{
|
||||
Assert.True(UniformChunk().IsUniform());
|
||||
|
||||
var varied = UniformChunk();
|
||||
varied.WalkZE[42] = 99;
|
||||
Assert.False(varied.IsUniform());
|
||||
|
||||
var strata = UniformChunk();
|
||||
var offsets = new ushort[StepChunk.CellsPerChunk];
|
||||
Array.Fill(offsets, StepChunk.NoStrata);
|
||||
offsets[0] = 0;
|
||||
strata.SetStrata(offsets, new byte[] { 0 });
|
||||
Assert.False(strata.IsUniform());
|
||||
|
||||
var swim = UniformChunk();
|
||||
swim.AllocateSwimLayer(); // a uniform-looking base but with a swim layer is NOT uniform
|
||||
Assert.False(swim.IsUniform());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Uniform_RoundTrips_Identically_AndIsCompact()
|
||||
{
|
||||
var src = UniformChunk(walk: 0xC1, wet: 0x00, z: 12, multis: 9);
|
||||
var src = FlatFullChunk();
|
||||
var rt = RoundTrip(src, 5, 6, out var fileLen);
|
||||
Assert.True(fileLen < 200, $"uniform .swb too large: {fileLen}");
|
||||
AssertChunksEqual(src, rt);
|
||||
// Full record with all 16 Z arrays elided: header(48) + ~783-byte record + index(20).
|
||||
// A v5 full record alone is > 5 KB, so a sub-1100-byte file proves elision fired.
|
||||
Assert.True(fileLen < 1100, $"expected all base Z arrays to elide; file was {fileLen} bytes");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SlopedSubset_OnlyVaryingArraysPresent_RoundTrips()
|
||||
{
|
||||
var flat = FlatFullChunk();
|
||||
var flatPath = Write1(flat, 1, 1);
|
||||
long flatLen;
|
||||
try { flatLen = new FileInfo(flatPath).Length; } finally { File.Delete(flatPath); }
|
||||
|
||||
// Bump WalkZN by +1 on cells walkable to the N (slope in one direction only) -> exactly
|
||||
// one base Z array (WalkZN, bit 0) becomes present; the other 15 still elide.
|
||||
var sloped = FlatFullChunk();
|
||||
for (var i = 0; i < StepChunk.CellsPerChunk; i++)
|
||||
{
|
||||
if ((sloped.WalkMask[i] & 1) != 0)
|
||||
{
|
||||
sloped.WalkZN[i] = (sbyte)(sloped.WalkZN[i] + 1);
|
||||
}
|
||||
}
|
||||
|
||||
var rt = RoundTrip(sloped, 2, 3, out var slopedLen);
|
||||
AssertChunksEqual(sloped, rt);
|
||||
Assert.True(slopedLen > flatLen, "one present array should grow the record vs all-flat");
|
||||
Assert.True(slopedLen <= flatLen + StepChunk.CellsPerChunk, "only one 256-byte residual array should be added");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
|
@ -152,6 +205,20 @@ public class StepCacheFileV5Tests
|
|||
AssertChunksEqual(src, RoundTrip(src, 1, 2, out _));
|
||||
}
|
||||
|
||||
// ---- shape coverage (Task 3) ----
|
||||
|
||||
[Fact]
|
||||
public void Coastline_NonzeroSourceZ_PartialWalkability_AllElide()
|
||||
{
|
||||
// FlatFullChunk already models a coastline: per-cell partial walk/wet masks, SourceZ != 0,
|
||||
// flat where walkable and 0 (baker default) where not. A plain SourceZ residual would emit
|
||||
// -SourceZ on every unwalkable direction; the masked predictor must drive ALL arrays to elide.
|
||||
var src = FlatFullChunk(multis: 2, baseZ: 25);
|
||||
var rt = RoundTrip(src, 7, 7, out var fileLen);
|
||||
AssertChunksEqual(src, rt);
|
||||
Assert.True(fileLen < 1100, $"masked predictor should elide every array on flat coastline; file was {fileLen} bytes");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SwimLayer_Full_RoundTrips_Identically()
|
||||
{
|
||||
|
|
@ -178,24 +245,10 @@ public class StepCacheFileV5Tests
|
|||
Assert.True(rt.StrataData.SequenceEqual(src.StrataData));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OlderVersion_IsRejected()
|
||||
{
|
||||
var path = Write1(UniformChunk(), 0, 0);
|
||||
try
|
||||
{
|
||||
var bytes = File.ReadAllBytes(path);
|
||||
bytes[4] = 4; bytes[5] = 0; bytes[6] = 0; bytes[7] = 0; // version 4 < MinSupportedVersion 5
|
||||
File.WriteAllBytes(path, bytes);
|
||||
Assert.Null(StepCacheFile.OpenForLazy(path));
|
||||
}
|
||||
finally { File.Delete(path); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SwimAndStrata_Full_RoundTrips_Identically()
|
||||
{
|
||||
// Exercises the combined trailer ordering: swim-layer trailer THEN strata trailer.
|
||||
// Combined trailer ordering: swim-layer trailer THEN strata trailer, after the residual blocks.
|
||||
var src = SwimChunk(multis: 11);
|
||||
var offsets = new ushort[StepChunk.CellsPerChunk];
|
||||
Array.Fill(offsets, StepChunk.NoStrata);
|
||||
|
|
@ -210,4 +263,18 @@ public class StepCacheFileV5Tests
|
|||
AssertChunksEqual(src, rt);
|
||||
Assert.True(rt.StrataData.SequenceEqual(src.StrataData));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OlderVersion_IsRejected()
|
||||
{
|
||||
var path = Write1(FlatFullChunk(), 0, 0);
|
||||
try
|
||||
{
|
||||
var bytes = File.ReadAllBytes(path);
|
||||
bytes[4] = 5; bytes[5] = 0; bytes[6] = 0; bytes[7] = 0; // version 5 < MinSupportedVersion 6
|
||||
File.WriteAllBytes(path, bytes);
|
||||
Assert.Null(StepCacheFile.OpenForLazy(path));
|
||||
}
|
||||
finally { File.Delete(path); }
|
||||
}
|
||||
}
|
||||
|
|
@ -14,11 +14,11 @@ namespace Server.Engines.Pathing.Cache;
|
|||
/// only when the cache asks for them. RAM stays bounded by MaxResidentChunks regardless
|
||||
/// of file size.
|
||||
///
|
||||
/// File layout v5 (little-endian, BufferWriter / BufferReader convention):
|
||||
/// File layout v6 (little-endian, BufferWriter / BufferReader convention):
|
||||
///
|
||||
/// Header (48 bytes):
|
||||
/// u32 Magic = 0x42575300 ('SWB\0')
|
||||
/// u32 Version = current FormatVersion (5)
|
||||
/// u32 Version = current FormatVersion (6)
|
||||
/// u32 MapId
|
||||
/// u64 Fingerprint XxHash3 over (1) LandTable + ItemTable flags AND (2) the
|
||||
/// on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul.
|
||||
|
|
@ -40,11 +40,16 @@ namespace Server.Engines.Pathing.Cache;
|
|||
/// // Full (Kind == 0) body:
|
||||
/// u8 HasStrata 0 = single-Z chunk (no strata trailer); 1 = strata trailer follows
|
||||
/// u8 HasSwimLayer 0 = no shore cells (no swim trailer); 1 = swim trailer follows
|
||||
/// u16 ZArrayMask bit d set => base directional Z array d is present below as a
|
||||
/// residual[256] block; cleared => array equals its prediction and is
|
||||
/// omitted (synthesized at read). bits 0-7 = WalkZ N..NW (predicted via
|
||||
/// WalkMask), bits 8-15 = SwimZ N..NW (predicted via WetMask).
|
||||
/// byte WalkMask[256]
|
||||
/// byte WetMask[256]
|
||||
/// sbyte SourceZ[256]
|
||||
/// sbyte WalkZN[256]..WalkZNW[256] (8 arrays in N,NE,E,SE,S,SW,W,NW order)
|
||||
/// sbyte SwimZN[256]..SwimZNW[256] (8 arrays in same order; baked at WALK source-Z)
|
||||
/// // For each d in 0..15 with ZArrayMask bit d set, in N,NE,E,SE,S,SW,W,NW order
|
||||
/// // (walk arrays first, then swim):
|
||||
/// sbyte residual_d[256] reconstruct: Z_d[c] = (mask bit set ? SourceZ[c] : 0) + residual_d[c]
|
||||
/// // Swim layer trailer — only when HasSwimLayer == 1 (chunks containing shore cells):
|
||||
/// sbyte SwimSourceZ[256] (NoSwimLayerCell sentinel = sbyte.MinValue)
|
||||
/// byte SwimMask[256] (per-cell swim mask baked at SwimSourceZ)
|
||||
|
|
@ -62,8 +67,10 @@ namespace Server.Engines.Pathing.Cache;
|
|||
/// Index trailer (20 × ChunkCount bytes):
|
||||
/// For each chunk: { u64 chunkKey, u64 fileOffset, u32 recordLength }
|
||||
///
|
||||
/// Per-chunk fixed portion: ~5,393 bytes. Strata trailer: 516 + N × ~30 bytes for a
|
||||
/// chunk with N multi-Z cells averaging ~2 strata each. LRU bookkeeping
|
||||
/// Per-chunk fixed portion (Kind + flags + ZArrayMask + WalkMask + WetMask + SourceZ):
|
||||
/// ~783 bytes; each present base Z array adds 256 bytes (0..16 present, so up to ~4 KB).
|
||||
/// A fully-flat Full chunk stores no residual blocks. Strata trailer: 516 + N × ~30 bytes
|
||||
/// for a chunk with N multi-Z cells averaging ~2 strata each. LRU bookkeeping
|
||||
/// (LastTouchedTicks) is intentionally not persisted.
|
||||
///
|
||||
/// Files with version < <see cref="MinSupportedVersion"/> are silently rejected
|
||||
|
|
@ -72,7 +79,7 @@ namespace Server.Engines.Pathing.Cache;
|
|||
internal static class StepCacheFile
|
||||
{
|
||||
public const uint Magic = 0x42575300; // 'SWB\0'
|
||||
public const uint FormatVersion = 5;
|
||||
public const uint FormatVersion = 6;
|
||||
|
||||
/// <summary>
|
||||
/// Lowest format version this binary can load. Files below this version are treated as
|
||||
|
|
@ -86,9 +93,12 @@ internal static class StepCacheFile
|
|||
/// one-time re-bake of stale v3 files on first boot under the new binary. Bumped to 5 for
|
||||
/// uniform-chunk elision: each record now begins with a Kind byte (0 = Full, 2 = Uniform);
|
||||
/// a fully-uniform chunk (no strata, no swim layer, all 19 base arrays constant) stores
|
||||
/// one cell's worth of data (~28 bytes) instead of the full record.
|
||||
/// one cell's worth of data (~28 bytes) instead of the full record. Bumped to 6 for
|
||||
/// predictive-Z residuals: each base directional Z array is stored as a masked residual
|
||||
/// against SourceZ (a ZArrayMask u16 flags which arrays are present); arrays matching their
|
||||
/// prediction are omitted and synthesized at read. v5 files are rejected and re-baked once.
|
||||
/// </summary>
|
||||
public const uint MinSupportedVersion = 5;
|
||||
public const uint MinSupportedVersion = 6;
|
||||
|
||||
// Per-chunk record discriminator (first byte after BuiltMultisVersion). 1 is reserved.
|
||||
private const byte KindFull = 0;
|
||||
|
|
@ -112,6 +122,7 @@ internal static class StepCacheFile
|
|||
private const int BytesPerChunkBase =
|
||||
sizeof(ushort) + sizeof(ushort) + sizeof(uint)
|
||||
+ sizeof(byte) + sizeof(byte) + sizeof(byte) // Kind + HasStrata + HasSwimLayer
|
||||
+ sizeof(ushort) // ZArrayMask
|
||||
+ StepChunk.CellsPerChunk // WalkMask
|
||||
+ StepChunk.CellsPerChunk // WetMask
|
||||
+ StepChunk.CellsPerChunk // SourceZ
|
||||
|
|
@ -372,6 +383,35 @@ internal static class StepCacheFile
|
|||
|
||||
private static ulong PackChunkKey(int chunkX, int chunkY) => ((ulong)(uint)chunkX << 32) | (uint)chunkY;
|
||||
|
||||
/// <summary>
|
||||
/// Predicted directional-Z for one cell/direction: the cell's own SourceZ when the
|
||||
/// direction is walkable/wet (mask bit set), else 0 — matching the baker, which leaves
|
||||
/// non-walkable directional slots at their zero-initialized default
|
||||
/// (StepProbe.ComputeMaskAt clears walkZs/swimZs and writes only on a successful step).
|
||||
/// </summary>
|
||||
internal static sbyte Predict(byte dirMaskByte, int bit, sbyte sourceZ) =>
|
||||
(dirMaskByte >> bit & 1) != 0 ? sourceZ : (sbyte)0;
|
||||
|
||||
/// <summary>
|
||||
/// Residual of an absolute directional-Z against its prediction. Unchecked two's-complement
|
||||
/// so the transform is byte-exact for ALL sbyte inputs (no value-range constraint).
|
||||
/// </summary>
|
||||
internal static sbyte EncodeResidual(sbyte z, sbyte predict) => unchecked((sbyte)(z - predict));
|
||||
|
||||
/// <summary>Inverse of <see cref="EncodeResidual"/>: absolute directional-Z = predict + residual.</summary>
|
||||
internal static sbyte DecodeZ(sbyte predict, sbyte residual) => unchecked((sbyte)(predict + residual));
|
||||
|
||||
/// <summary>
|
||||
/// The 16 base directional-Z arrays in canonical order: walk N..NW (indices 0-7),
|
||||
/// then swim N..NW (8-15). Index d uses WalkMask (d < 8) or WetMask (d >= 8)
|
||||
/// with direction bit (d & 7). Allocates a 16-slot reference array (bake/read time only).
|
||||
/// </summary>
|
||||
private static sbyte[][] GetBaseZArrays(StepChunk c) => new[]
|
||||
{
|
||||
c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW,
|
||||
c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW,
|
||||
};
|
||||
|
||||
private static void WriteChunk(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
|
||||
{
|
||||
w.Write((ushort)chunkX);
|
||||
|
|
@ -414,27 +454,47 @@ internal static class StepCacheFile
|
|||
w.Write((byte)(hasStrata ? 1 : 0));
|
||||
w.Write((byte)(hasSwimLayer ? 1 : 0));
|
||||
|
||||
// Predictive-Z: each base directional Z array is stored as a masked residual against
|
||||
// SourceZ. Bit d of ZArrayMask is set only when array d differs from its prediction
|
||||
// somewhere; cleared arrays are omitted and rebuilt from mask+SourceZ at read.
|
||||
var zArrays = GetBaseZArrays(chunk);
|
||||
ushort zArrayMask = 0;
|
||||
for (var d = 0; d < 16; d++)
|
||||
{
|
||||
var z = zArrays[d];
|
||||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||||
var bit = d & 7;
|
||||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||||
{
|
||||
if (z[cell] != Predict(dirMask[cell], bit, chunk.SourceZ[cell]))
|
||||
{
|
||||
zArrayMask |= (ushort)(1 << d);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Write(zArrayMask);
|
||||
|
||||
w.Write(chunk.WalkMask);
|
||||
w.Write(chunk.WetMask);
|
||||
WriteSBytes(w, chunk.SourceZ);
|
||||
|
||||
WriteSBytes(w, chunk.WalkZN);
|
||||
WriteSBytes(w, chunk.WalkZNE);
|
||||
WriteSBytes(w, chunk.WalkZE);
|
||||
WriteSBytes(w, chunk.WalkZSE);
|
||||
WriteSBytes(w, chunk.WalkZS);
|
||||
WriteSBytes(w, chunk.WalkZSW);
|
||||
WriteSBytes(w, chunk.WalkZW);
|
||||
WriteSBytes(w, chunk.WalkZNW);
|
||||
|
||||
WriteSBytes(w, chunk.SwimZN);
|
||||
WriteSBytes(w, chunk.SwimZNE);
|
||||
WriteSBytes(w, chunk.SwimZE);
|
||||
WriteSBytes(w, chunk.SwimZSE);
|
||||
WriteSBytes(w, chunk.SwimZS);
|
||||
WriteSBytes(w, chunk.SwimZSW);
|
||||
WriteSBytes(w, chunk.SwimZW);
|
||||
WriteSBytes(w, chunk.SwimZNW);
|
||||
Span<sbyte> residual = stackalloc sbyte[StepChunk.CellsPerChunk];
|
||||
for (var d = 0; d < 16; d++)
|
||||
{
|
||||
if ((zArrayMask >> d & 1) == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
var z = zArrays[d];
|
||||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||||
var bit = d & 7;
|
||||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||||
{
|
||||
residual[cell] = EncodeResidual(z[cell], Predict(dirMask[cell], bit, chunk.SourceZ[cell]));
|
||||
}
|
||||
w.Write(MemoryMarshal.Cast<sbyte, byte>(residual));
|
||||
}
|
||||
|
||||
if (hasSwimLayer)
|
||||
{
|
||||
|
|
@ -503,28 +563,37 @@ internal static class StepCacheFile
|
|||
|
||||
var hasStrata = r.ReadByte() != 0;
|
||||
var hasSwimLayer = r.ReadByte() != 0;
|
||||
var zArrayMask = r.ReadUShort();
|
||||
|
||||
r.Read(chunk.WalkMask);
|
||||
r.Read(chunk.WetMask);
|
||||
ReadSBytes(r, chunk.SourceZ);
|
||||
|
||||
ReadSBytes(r, chunk.WalkZN);
|
||||
ReadSBytes(r, chunk.WalkZNE);
|
||||
ReadSBytes(r, chunk.WalkZE);
|
||||
ReadSBytes(r, chunk.WalkZSE);
|
||||
ReadSBytes(r, chunk.WalkZS);
|
||||
ReadSBytes(r, chunk.WalkZSW);
|
||||
ReadSBytes(r, chunk.WalkZW);
|
||||
ReadSBytes(r, chunk.WalkZNW);
|
||||
|
||||
ReadSBytes(r, chunk.SwimZN);
|
||||
ReadSBytes(r, chunk.SwimZNE);
|
||||
ReadSBytes(r, chunk.SwimZE);
|
||||
ReadSBytes(r, chunk.SwimZSE);
|
||||
ReadSBytes(r, chunk.SwimZS);
|
||||
ReadSBytes(r, chunk.SwimZSW);
|
||||
ReadSBytes(r, chunk.SwimZW);
|
||||
ReadSBytes(r, chunk.SwimZNW);
|
||||
// Predictive-Z reconstruction: present arrays carry residuals (z = predict + residual);
|
||||
// absent arrays are synthesized from mask+SourceZ (z = predict, residual implicitly 0).
|
||||
var zArrays = GetBaseZArrays(chunk);
|
||||
Span<sbyte> residual = stackalloc sbyte[StepChunk.CellsPerChunk];
|
||||
for (var d = 0; d < 16; d++)
|
||||
{
|
||||
var z = zArrays[d];
|
||||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||||
var bit = d & 7;
|
||||
if ((zArrayMask >> d & 1) != 0)
|
||||
{
|
||||
r.Read(MemoryMarshal.Cast<sbyte, byte>(residual));
|
||||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||||
{
|
||||
z[cell] = DecodeZ(Predict(dirMask[cell], bit, chunk.SourceZ[cell]), residual[cell]);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||||
{
|
||||
z[cell] = Predict(dirMask[cell], bit, chunk.SourceZ[cell]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hasSwimLayer)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -208,13 +208,19 @@ whole-file) and bounded RAM (only touched chunks materialize, LRU-capped):
|
|||
mask + Z is stored as a few bytes (flag + mask + Z) instead of 5,393. Pairs with a two-level
|
||||
**sector index** so a uniform super-sector (open ocean) collapses to a single entry — UO's
|
||||
16-tile sectors nest cleanly inside.
|
||||
2. **Predictive (lossless) Z residuals** (kills the 76% that is the 16 directional Z arrays).
|
||||
Predict each `WalkZ_dir`/`SwimZ_dir` from the cell's **own `SourceZ`** and store only the
|
||||
residual: 0 on flat ground (→ compresses to nothing → effectively just the walk *bit*),
|
||||
±1..3 on slopes/stairs, larger or strata for bridges/multi-Z. Reconstruct
|
||||
`WalkZ_dir = SourceZ + residual` at load → byte-identical in-memory `StepChunk`, so **no
|
||||
runtime recompute and no risk of diverging from `MovementImpl`** (the bugs the cache exists
|
||||
to avoid). Predict from *self* (not a neighbor) to keep chunk independence.
|
||||
2. **Predictive (lossless) Z residuals** — *shipped as format v6.* Kills the bulk of the 16
|
||||
base directional Z arrays in Full chunks. Each array is stored as a **masked residual**
|
||||
against the cell's **own `SourceZ`**: predict `mask bit ? SourceZ : 0` (the mask term matches
|
||||
the baker, which leaves non-walkable directional slots at `0`), residual `= Z − predict` via
|
||||
unchecked two's-complement (byte-exact for all inputs). A new u16 `ZArrayMask` flags which of
|
||||
the 16 arrays differ from their prediction; an array that matches everywhere (flat terrain —
|
||||
including partial-walkability coastlines with nonzero `SourceZ`) is **omitted entirely** and
|
||||
synthesized from `mask + SourceZ` at read. Reconstruct `Z = predict + residual` → byte-identical
|
||||
in-memory `StepChunk`, so **no runtime recompute and no risk of diverging from `MovementImpl`**.
|
||||
Self-prediction (not a neighbor) keeps chunk independence. The residual *subtraction* itself
|
||||
saves ~0 bytes (a residual is still 1 byte/cell); the win is the per-array elision, and leaving
|
||||
non-elided arrays in residual form makes #3 a pure codec add (no further transform/format bump).
|
||||
Swim-layer + strata trailers stay absolute, deferred to #3.
|
||||
3. **Per-chunk block compression + index compaction.** zstd/deflate each chunk record
|
||||
independently (index already carries a per-chunk `length`; reader decompresses one chunk on
|
||||
read). At 256-cell granularity the **index overhead** then dominates (20 B/chunk ×
|
||||
|
|
@ -236,4 +242,12 @@ Validate size **and** read-latency vs the corpus in the benchmark repo after eac
|
|||
(both stay Full). **#1 alone: 592.2 MB → 231.9 MB (−61%), actual on-disk.** The residual is
|
||||
~150 MB of non-uniform land Z-blocks (→ #2 predictive-Z) + ~81 MB of swim-layer trailers
|
||||
(→ #2/#3). Confirms build order **#1 → #2 → #3**, with #1 the dominant, lowest-risk,
|
||||
no-algorithm-change win (now shipped as format v5).
|
||||
no-algorithm-change win (shipped as format v5).
|
||||
- *Calibrated v6 (`BakeMap`-measured, full Trammel, `MaxResidentChunks` raised above 114,688):*
|
||||
**#2 predictive-Z: 231.9 MB → 124.7 MB (−46%, −107 MB; −78% vs the original 565 MB).** Of the
|
||||
42,775 Full chunks (71,913 are Uniform), **walk directional-Z arrays elide 47.7%** and **swim
|
||||
arrays elide 87.6%** (per-array, not per-cell — one slope cell keeps a 256-byte array, which is
|
||||
why the per-array rate trails the 95.2% per-cell zero-residual). Remaining v6 bytes are dominated
|
||||
by present walk-residual blocks (~46 MB, mostly ±1..3 + zeros → highly compressible), the
|
||||
per-Full-chunk mask/SourceZ base (~33 MB), and absolute swim-layer trailers (~26 MB) — all prime
|
||||
targets for #3 per-chunk compression.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue