feat(pathfinding): .swb format v7 per-chunk compression (#3a) (#2470)
## Summary Phase #3a, stacked on #2469. Compresses each chunk record independently with libdeflate (random access preserved). Trammel: 124.7 MB → 19.2 MB (−85%). ## Details - Whole-record framing `[u32 UncompressedLen][payload]`; records that don't shrink (tiny Uniform) are stored raw, detected as payload length == UncompressedLen. - Codec chosen by full-Trammel spike: libdeflate VeryHigh (16.5 MB, 1.83 µs/chunk decompress) over zstd L19/22 (17.4 MB) and managed Brotli q11 (16.4 MB) — best native ratio, fastest decompress, already the repo's packet codec (no new dependency). - Reuses cached thread-static bindings: `Deflate.Maximum` for bake, `Deflate.Standard` for reads. - Compression is bake-time only; decompression is one-time per chunk (LRU-cached). - Format v7; v6 files rejected and re-baked once. ## Tests v7 unit tests + the v6 suite run through the compression path; full pathfinding suite green; Release build clean.
This commit is contained in:
Kamron Batman
GitHub
parent
94537f83f8
commit
c265bcbb5e
4 changed files with 255 additions and 15 deletions
|
|
@ -23,5 +23,13 @@ public static class Deflate
|
|||
[ThreadStatic]
|
||||
private static LibDeflateBinding _standard;
|
||||
|
||||
[ThreadStatic]
|
||||
private static LibDeflateBinding _maximum;
|
||||
|
||||
public static LibDeflateBinding Standard => _standard ??= new LibDeflateBinding();
|
||||
|
||||
// Best-ratio compressor. Construction allocates a native libdeflate compressor, so it is
|
||||
// cached per thread like Standard and reused. Decompression is level-independent, so the
|
||||
// decompress path can use either accessor. libdeflate is not thread-safe — hence ThreadStatic.
|
||||
public static LibDeflateBinding Maximum => _maximum ??= new LibDeflateBinding(LibDeflateCompressionLevel.VeryHigh);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
using Server.Engines.Pathing.Cache;
|
||||
using Xunit;
|
||||
|
||||
namespace Server.Tests.Pathfinding;
|
||||
|
||||
// v7 = per-chunk libdeflate compression on top of the v6 predictive-Z format. Each record is
|
||||
// compressed independently (random access preserved) behind a u32 uncompressed-length prefix;
|
||||
// records that do not shrink (tiny Uniform records) are stored raw. StepCacheFileV6Tests already
|
||||
// round-trips every chunk shape through this compression path; these tests cover the v7-specific
|
||||
// behavior: that compression engages, that the stored-raw fallback round-trips, and v6 rejection.
|
||||
[Collection("Sequential Pathfinding Tests")]
|
||||
public class StepCacheFileV7Tests
|
||||
{
|
||||
private static StepChunk VariedChunk(int multis = 3)
|
||||
{
|
||||
var c = new StepChunk { BuiltMultisVersion = multis };
|
||||
for (var i = 0; i < StepChunk.CellsPerChunk; i++)
|
||||
{
|
||||
c.WalkMask[i] = (byte)(i & 0xFF);
|
||||
c.WetMask[i] = (byte)((i * 7) & 0xFF);
|
||||
c.SourceZ[i] = (sbyte)((i % 40) - 20);
|
||||
c.WalkZN[i] = (sbyte)(c.SourceZ[i] + (i % 3));
|
||||
c.SwimZS[i] = (sbyte)(c.SourceZ[i] - (i % 2));
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
private static StepChunk UniformChunk(byte walk = 0xC1, sbyte z = 10, int multis = 7)
|
||||
{
|
||||
var c = new StepChunk { BuiltMultisVersion = multis };
|
||||
Array.Fill(c.WalkMask, walk);
|
||||
Array.Fill(c.SourceZ, z);
|
||||
foreach (var arr in new[]
|
||||
{
|
||||
c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW,
|
||||
c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW
|
||||
})
|
||||
{
|
||||
Array.Fill(arr, z);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
private static string Write1(StepChunk c, int cx, int cy)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), $"swbv7_{Guid.NewGuid():N}.swb");
|
||||
var emitted = false;
|
||||
StepCacheFile.Write(path, 1u, 1u, (out int ox, out int oy, out StepChunk oc) =>
|
||||
{
|
||||
if (emitted) { ox = oy = 0; oc = null!; return false; }
|
||||
emitted = true; ox = cx; oy = cy; oc = c; return true;
|
||||
});
|
||||
return path;
|
||||
}
|
||||
|
||||
private static StepChunk RoundTrip(StepChunk src, int cx, int cy, out long fileLen)
|
||||
{
|
||||
var path = Write1(src, cx, cy);
|
||||
try
|
||||
{
|
||||
fileLen = new FileInfo(path).Length;
|
||||
using var reader = StepCacheFile.OpenForLazy(path);
|
||||
Assert.NotNull(reader);
|
||||
var rt = reader!.TryReadChunk(cx, cy);
|
||||
Assert.NotNull(rt);
|
||||
return rt!;
|
||||
}
|
||||
finally { File.Delete(path); }
|
||||
}
|
||||
|
||||
private static void AssertBaseEqual(StepChunk a, StepChunk b)
|
||||
{
|
||||
Assert.Equal(a.BuiltMultisVersion, b.BuiltMultisVersion);
|
||||
Assert.True(a.WalkMask.AsSpan().SequenceEqual(b.WalkMask));
|
||||
Assert.True(a.WetMask.AsSpan().SequenceEqual(b.WetMask));
|
||||
Assert.True(a.SourceZ.AsSpan().SequenceEqual(b.SourceZ));
|
||||
var az = new[] { a.WalkZN, a.WalkZNE, a.WalkZE, a.WalkZSE, a.WalkZS, a.WalkZSW, a.WalkZW, a.WalkZNW,
|
||||
a.SwimZN, a.SwimZNE, a.SwimZE, a.SwimZSE, a.SwimZS, a.SwimZSW, a.SwimZW, a.SwimZNW };
|
||||
var bz = new[] { b.WalkZN, b.WalkZNE, b.WalkZE, b.WalkZSE, b.WalkZS, b.WalkZSW, b.WalkZW, b.WalkZNW,
|
||||
b.SwimZN, b.SwimZNE, b.SwimZE, b.SwimZSE, b.SwimZS, b.SwimZSW, b.SwimZW, b.SwimZNW };
|
||||
for (var i = 0; i < az.Length; i++)
|
||||
{
|
||||
Assert.True(az[i].AsSpan().SequenceEqual(bz[i]), $"base Z array {i} differs");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Varied_Compresses_AndRoundTrips()
|
||||
{
|
||||
var src = VariedChunk(multis: 4);
|
||||
var rt = RoundTrip(src, 1, 2, out var fileLen);
|
||||
AssertBaseEqual(src, rt);
|
||||
// The uncompressed v6 Full record for a varied chunk is > 5 KB. Compressed + header(48)
|
||||
// + index(20), the whole file must be well under that — proving compression engaged.
|
||||
Assert.True(fileLen < 4000, $"expected compression to shrink the record; file was {fileLen} bytes");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Uniform_StoredRaw_RoundTrips()
|
||||
{
|
||||
// A Uniform record body is ~24 bytes; libdeflate cannot shrink it, so WriteChunk stores it
|
||||
// raw (payload length == uncompressed length). The reader must take the raw path and rebuild.
|
||||
var src = UniformChunk(walk: 0xC1, z: 12, multis: 9);
|
||||
var rt = RoundTrip(src, 5, 6, out var fileLen);
|
||||
AssertBaseEqual(src, rt);
|
||||
Assert.True(fileLen < 200, $"uniform record should stay tiny; file was {fileLen} bytes");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void V6_IsRejected()
|
||||
{
|
||||
var path = Write1(UniformChunk(), 0, 0);
|
||||
try
|
||||
{
|
||||
var bytes = File.ReadAllBytes(path);
|
||||
bytes[4] = 6; bytes[5] = 0; bytes[6] = 0; bytes[7] = 0; // version 6 < MinSupportedVersion 7
|
||||
File.WriteAllBytes(path, bytes);
|
||||
Assert.Null(StepCacheFile.OpenForLazy(path));
|
||||
}
|
||||
finally { File.Delete(path); }
|
||||
}
|
||||
}
|
||||
|
|
@ -2,7 +2,9 @@ using System;
|
|||
using System.Buffers.Binary;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Runtime.InteropServices;
|
||||
using Server.Compression;
|
||||
|
||||
namespace Server.Engines.Pathing.Cache;
|
||||
|
||||
|
|
@ -14,11 +16,11 @@ namespace Server.Engines.Pathing.Cache;
|
|||
/// only when the cache asks for them. RAM stays bounded by MaxResidentChunks regardless
|
||||
/// of file size.
|
||||
///
|
||||
/// File layout v6 (little-endian, BufferWriter / BufferReader convention):
|
||||
/// File layout v7 (little-endian, BufferWriter / BufferReader convention):
|
||||
///
|
||||
/// Header (48 bytes):
|
||||
/// u32 Magic = 0x42575300 ('SWB\0')
|
||||
/// u32 Version = current FormatVersion (6)
|
||||
/// u32 Version = current FormatVersion (7)
|
||||
/// u32 MapId
|
||||
/// u64 Fingerprint XxHash3 over (1) LandTable + ItemTable flags AND (2) the
|
||||
/// on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul.
|
||||
|
|
@ -31,6 +33,13 @@ namespace Server.Engines.Pathing.Cache;
|
|||
/// u64 IndexOffset File position where the chunk index begins.
|
||||
///
|
||||
/// Per chunk (ChunkCount times, variable size):
|
||||
/// u32 UncompressedLen Size of the inflated record body below.
|
||||
/// byte[] Payload The record body (the v6 layout that follows), libdeflate-
|
||||
/// compressed. If the on-disk payload length (index recordLength −
|
||||
/// 4) equals UncompressedLen, the body was stored raw because
|
||||
/// compression did not shrink it (tiny Uniform records).
|
||||
///
|
||||
/// Record body (after inflate — the v6 layout):
|
||||
/// u16 ChunkX
|
||||
/// u16 ChunkY
|
||||
/// u32 BuiltMultisVersion
|
||||
|
|
@ -79,7 +88,7 @@ namespace Server.Engines.Pathing.Cache;
|
|||
internal static class StepCacheFile
|
||||
{
|
||||
public const uint Magic = 0x42575300; // 'SWB\0'
|
||||
public const uint FormatVersion = 6;
|
||||
public const uint FormatVersion = 7;
|
||||
|
||||
/// <summary>
|
||||
/// Lowest format version this binary can load. Files below this version are treated as
|
||||
|
|
@ -97,8 +106,11 @@ internal static class StepCacheFile
|
|||
/// predictive-Z residuals: each base directional Z array is stored as a masked residual
|
||||
/// against SourceZ (a ZArrayMask u16 flags which arrays are present); arrays matching their
|
||||
/// prediction are omitted and synthesized at read. v5 files are rejected and re-baked once.
|
||||
/// Bumped to 7 for per-chunk compression: each chunk record is libdeflate-compressed
|
||||
/// independently (random access preserved) behind a u32 uncompressed-length prefix; tiny
|
||||
/// records that do not shrink are stored raw. v6 files are rejected and re-baked once.
|
||||
/// </summary>
|
||||
public const uint MinSupportedVersion = 6;
|
||||
public const uint MinSupportedVersion = 7;
|
||||
|
||||
// Per-chunk record discriminator (first byte after BuiltMultisVersion). 1 is reserved.
|
||||
private const byte KindFull = 0;
|
||||
|
|
@ -255,6 +267,15 @@ internal static class StepCacheFile
|
|||
w.Write(chunkCount);
|
||||
w.Write(0UL); // IndexOffset placeholder, patched after chunks
|
||||
|
||||
// Per-chunk compression: each record is built uncompressed into `recordScratch`, then
|
||||
// libdeflate-compressed into `compScratch` and framed as [u32 uncompressedLen][payload].
|
||||
// Reuse the cached per-thread VeryHigh compressor (construction allocates native state).
|
||||
// libdeflate is not thread-safe, but bake runs single-threaded. VeryHigh is the best-ratio
|
||||
// level and its slow compression is irrelevant offline (decompression is what's hot, ~1.8us).
|
||||
var packer = Deflate.Maximum;
|
||||
var recordScratch = new byte[BytesPerChunkBase + 1024];
|
||||
var compScratch = new byte[packer.MaxPackSize(recordScratch.Length)];
|
||||
|
||||
var indexEntries = new (ulong key, ulong offset, uint length)[chunkCount];
|
||||
var written = 0u;
|
||||
while (next(out var chunkX, out var chunkY, out var chunk))
|
||||
|
|
@ -266,7 +287,7 @@ internal static class StepCacheFile
|
|||
);
|
||||
}
|
||||
var chunkOffset = (ulong)w.Position;
|
||||
WriteChunk(w, chunkX, chunkY, chunk);
|
||||
WriteChunk(w, chunkX, chunkY, chunk, packer, ref recordScratch, ref compScratch);
|
||||
var chunkLength = (uint)((ulong)w.Position - chunkOffset);
|
||||
indexEntries[written] = (PackChunkKey(chunkX, chunkY), chunkOffset, chunkLength);
|
||||
written++;
|
||||
|
|
@ -412,7 +433,44 @@ internal static class StepCacheFile
|
|||
c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW,
|
||||
};
|
||||
|
||||
private static void WriteChunk(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
|
||||
/// <summary>
|
||||
/// Builds the uncompressed v6 record for one chunk into <paramref name="w"/>, libdeflate-
|
||||
/// compresses it, and writes it framed as [u32 uncompressedLen][payload]. The payload is the
|
||||
/// compressed bytes, or — when compression does not shrink the record (tiny Uniform records) —
|
||||
/// the raw record itself; the reader distinguishes the two by payload length vs uncompressedLen.
|
||||
/// </summary>
|
||||
private static void WriteChunk(
|
||||
BufferWriter w, int chunkX, int chunkY, StepChunk chunk,
|
||||
LibDeflateBinding packer, ref byte[] recordScratch, ref byte[] compScratch
|
||||
)
|
||||
{
|
||||
var rw = new BufferWriter(recordScratch, prefixStr: false);
|
||||
BuildRecord(rw, chunkX, chunkY, chunk);
|
||||
recordScratch = rw.Buffer; // may have grown; keep the larger buffer for reuse
|
||||
var recordLen = (int)rw.Position;
|
||||
|
||||
var bound = packer.MaxPackSize(recordLen);
|
||||
if (compScratch.Length < bound)
|
||||
{
|
||||
compScratch = new byte[bound];
|
||||
}
|
||||
|
||||
var compLen = packer.Pack(compScratch, recordScratch.AsSpan(0, recordLen));
|
||||
|
||||
w.Write((uint)recordLen);
|
||||
if (compLen > 0 && compLen < recordLen)
|
||||
{
|
||||
w.Write(compScratch.AsSpan(0, compLen));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Incompressible (or expanded): store the record raw. The reader detects this when
|
||||
// the on-disk payload length equals the uncompressed length.
|
||||
w.Write(recordScratch.AsSpan(0, recordLen));
|
||||
}
|
||||
}
|
||||
|
||||
private static void BuildRecord(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
|
||||
{
|
||||
w.Write((ushort)chunkX);
|
||||
w.Write((ushort)chunkY);
|
||||
|
|
@ -644,7 +702,8 @@ internal static class StepCacheFile
|
|||
{
|
||||
private FileStream _stream;
|
||||
private readonly Dictionary<ulong, (ulong offset, uint length)> _offsets;
|
||||
private byte[] _buffer;
|
||||
private byte[] _buffer; // raw on-disk record: [u32 uncompressedLen][payload]
|
||||
private byte[] _bodyBuffer; // decompressed v6 record, parsed by ReadChunk
|
||||
|
||||
public uint MapId { get; }
|
||||
public ulong Fingerprint { get; }
|
||||
|
|
@ -679,6 +738,7 @@ internal static class StepCacheFile
|
|||
ChunkCount = chunkCount;
|
||||
_offsets = offsets;
|
||||
_buffer = new byte[BytesPerChunkBase];
|
||||
_bodyBuffer = new byte[BytesPerChunkBase];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -699,8 +759,7 @@ internal static class StepCacheFile
|
|||
return null;
|
||||
}
|
||||
|
||||
// Grow the scratch buffer if this chunk's record is larger than what we have.
|
||||
// Common case: chunks fit in BytesPerChunkBase; only multi-Z-heavy chunks grow.
|
||||
// Grow the on-disk scratch buffer if this chunk's record is larger than what we have.
|
||||
if (entry.length > _buffer.Length)
|
||||
{
|
||||
_buffer = new byte[entry.length];
|
||||
|
|
@ -708,7 +767,41 @@ internal static class StepCacheFile
|
|||
|
||||
_stream.Position = (long)entry.offset;
|
||||
var read = _stream.Read(_buffer, 0, (int)entry.length);
|
||||
return read < (int)entry.length ? null : ReadChunk(_buffer);
|
||||
if (read < (int)entry.length || entry.length < sizeof(uint))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// Frame: [u32 uncompressedLen][payload]. payload is libdeflate-compressed, unless its
|
||||
// length equals uncompressedLen, in which case it was stored raw (incompressible).
|
||||
var uncompressedLen = (int)BinaryPrimitives.ReadUInt32LittleEndian(_buffer);
|
||||
var payloadLen = (int)entry.length - sizeof(uint);
|
||||
if (_bodyBuffer.Length < uncompressedLen)
|
||||
{
|
||||
_bodyBuffer = new byte[uncompressedLen];
|
||||
}
|
||||
|
||||
if (payloadLen == uncompressedLen)
|
||||
{
|
||||
Array.Copy(_buffer, sizeof(uint), _bodyBuffer, 0, uncompressedLen);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Decompression is level-independent, so reuse the shared per-thread binding
|
||||
// rather than allocating a native decompressor per reader. The cache is read on
|
||||
// the single game thread; libdeflate's non-thread-safety is satisfied by ThreadStatic.
|
||||
var result = Deflate.Standard.Unpack(
|
||||
_bodyBuffer.AsSpan(0, uncompressedLen),
|
||||
_buffer.AsSpan(sizeof(uint), payloadLen),
|
||||
out var produced
|
||||
);
|
||||
if (result != LibDeflateResult.Success || produced != uncompressedLen)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return ReadChunk(_bodyBuffer);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
|
|
@ -716,6 +809,7 @@ internal static class StepCacheFile
|
|||
_stream?.Dispose();
|
||||
_stream = null;
|
||||
_buffer = null;
|
||||
_bodyBuffer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -221,11 +221,20 @@ whole-file) and bounded RAM (only touched chunks materialize, LRU-capped):
|
|||
saves ~0 bytes (a residual is still 1 byte/cell); the win is the per-array elision, and leaving
|
||||
non-elided arrays in residual form makes #3 a pure codec add (no further transform/format bump).
|
||||
Swim-layer + strata trailers stay absolute, deferred to #3.
|
||||
3. **Per-chunk block compression + index compaction.** zstd/deflate each chunk record
|
||||
independently (index already carries a per-chunk `length`; reader decompresses one chunk on
|
||||
read). At 256-cell granularity the **index overhead** then dominates (20 B/chunk ×
|
||||
114 K ≈ 2.3 MB), so compact it in the same pass: chunks in fixed sweep order → implicit keys,
|
||||
delta-encoded offsets, lengths derivable → ~2–4 B/chunk.
|
||||
3. **Per-chunk block compression + index compaction.**
|
||||
- **#3a — per-chunk compression: *shipped as format v7.*** Each record is libdeflate-compressed
|
||||
independently (random access preserved; the reader inflates one chunk on read into a reused
|
||||
buffer) behind a `u32 UncompressedLen` frame; tiny Uniform records that do not shrink are
|
||||
stored raw (detected as on-disk payload length == UncompressedLen). Codec chosen by full-Trammel
|
||||
spike: **libdeflate VeryHigh** beat zstd L19/22 (17.4 MB) and tied managed Brotli q11 (16.4 MB)
|
||||
at **16.5 MB of compressed records**, with the *fastest* decompress (**1.83 µs/chunk**, vs zstd
|
||||
2.21) — and it is already the repo's packet codec (`LibDeflate.Bindings`), so no new dependency.
|
||||
zstd's large-window advantage doesn't apply at 256-cell record granularity. Compression runs at
|
||||
bake time only (offline); decompression is one-time per chunk (LRU-cached after). **Calibrated:
|
||||
v6 124.7 MB → v7 19.2 MB (−85%); −97% vs the original 565 MB.**
|
||||
- **#3b — index compaction (remaining, ~2 MB).** The v7 file's residual is the still-uncompacted
|
||||
index (20 B/chunk × 114 K ≈ 2.3 MB). Compact it: chunks in fixed sweep order → implicit keys,
|
||||
delta-encoded offsets, lengths derivable → ~3 B/chunk ≈ 0.34 MB, landing the file at ~17 MB.
|
||||
|
||||
**Do first:** a uniformity audit over a baked facet (how many chunks are fully uniform / have
|
||||
all-zero Z residuals?) to size the #1/#2 win before writing any format code. Each technique is a
|
||||
|
|
@ -251,3 +260,8 @@ Validate size **and** read-latency vs the corpus in the benchmark repo after eac
|
|||
by present walk-residual blocks (~46 MB, mostly ±1..3 + zeros → highly compressible), the
|
||||
per-Full-chunk mask/SourceZ base (~33 MB), and absolute swim-layer trailers (~26 MB) — all prime
|
||||
targets for #3 per-chunk compression.
|
||||
- *Calibrated v7 (`BakeMap`-measured, full Trammel):* **#3a per-chunk libdeflate VeryHigh: 124.7 MB
|
||||
→ 19.2 MB (−85%); −97% vs the original 565 MB.** Codec spike (per-chunk over the 122.4 MB of v6
|
||||
records): libdeflate VeryHigh **16.5 MB** vs Brotli q11 16.4 / zstd L19–22 17.4; decompress
|
||||
**1.83 µs/chunk** (libdeflate) vs 2.21 (zstd). The remaining ~2.3 MB is the uncompacted index
|
||||
(→ #3b).
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue