c265bcbb5e
## Summary Phase #3a, stacked on #2469. Compresses each chunk record independently with libdeflate (random access preserved). Trammel: 124.7 MB → 19.2 MB (−85%). ## Details - Whole-record framing `[u32 UncompressedLen][payload]`; records that don't shrink (tiny Uniform) are stored raw, detected as payload length == UncompressedLen. - Codec chosen by full-Trammel spike: libdeflate VeryHigh (16.5 MB, 1.83 µs/chunk decompress) over zstd L19/22 (17.4 MB) and managed Brotli q11 (16.4 MB) — best native ratio, fastest decompress, already the repo's packet codec (no new dependency). - Reuses cached thread-static bindings: `Deflate.Maximum` for bake, `Deflate.Standard` for reads. - Compression is bake-time only; decompression is one-time per chunk (LRU-cached). - Format v7; v6 files rejected and re-baked once. ## Tests v7 unit tests + the v6 suite run through the compression path; full pathfinding suite green; Release build clean.
815 lines
35 KiB
C#
815 lines
35 KiB
C#
using System;
|
||
using System.Buffers.Binary;
|
||
using System.Collections.Generic;
|
||
using System.IO;
|
||
using System.IO.Compression;
|
||
using System.Runtime.InteropServices;
|
||
using Server.Compression;
|
||
|
||
namespace Server.Engines.Pathing.Cache;
|
||
|
||
/// <summary>
|
||
/// Binary serializer + lazy reader for the step cache. Persists chunk records to disk
|
||
/// so a server warm-starts without paying chunk-build cost on the first pathfind through
|
||
/// a region. Lazy: opening a file reads only the header + chunk-offset index (~few KB
|
||
/// for tens of thousands of chunks), then individual chunks are seeked + deserialized
|
||
/// only when the cache asks for them. RAM stays bounded by MaxResidentChunks regardless
|
||
/// of file size.
|
||
///
|
||
/// File layout v7 (little-endian, BufferWriter / BufferReader convention):
|
||
///
|
||
/// Header (48 bytes):
|
||
/// u32 Magic = 0x42575300 ('SWB\0')
|
||
/// u32 Version = current FormatVersion (7)
|
||
/// u32 MapId
|
||
/// u64 Fingerprint XxHash3 over (1) LandTable + ItemTable flags AND (2) the
|
||
/// on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul.
|
||
/// Rejects a load when EITHER tile flags shifted (client patch)
|
||
/// OR the map data was rewritten (CentredSharp / UOFiddler edit).
|
||
/// The .mul format has no built-in CRC; this is the only way
|
||
/// to detect those mutations.
|
||
/// u64 BakeTimestamp DateTime.UtcNow.Ticks at write time (informational).
|
||
/// u32 ChunkCount
|
||
/// u64 IndexOffset File position where the chunk index begins.
|
||
///
|
||
/// Per chunk (ChunkCount times, variable size):
|
||
/// u32 UncompressedLen Size of the inflated record body below.
|
||
/// byte[] Payload The record body (the v6 layout that follows), libdeflate-
|
||
/// compressed. If the on-disk payload length (index recordLength −
|
||
/// 4) equals UncompressedLen, the body was stored raw because
|
||
/// compression did not shrink it (tiny Uniform records).
|
||
///
|
||
/// Record body (after inflate — the v6 layout):
|
||
/// u16 ChunkX
|
||
/// u16 ChunkY
|
||
/// u32 BuiltMultisVersion
|
||
/// u8 Kind 0 = Full; 2 = Uniform
|
||
/// // Uniform (Kind == 2): ~28-byte record — all 256 cells share these single values:
|
||
/// byte walkMask, wetMask; sbyte sourceZ; sbyte walkZ_N..NW (8); sbyte swimZ_N..NW (8)
|
||
/// // Full (Kind == 0) body:
|
||
/// u8 HasStrata 0 = single-Z chunk (no strata trailer); 1 = strata trailer follows
|
||
/// u8 HasSwimLayer 0 = no shore cells (no swim trailer); 1 = swim trailer follows
|
||
/// u16 ZArrayMask bit d set => base directional Z array d is present below as a
|
||
/// residual[256] block; cleared => array equals its prediction and is
|
||
/// omitted (synthesized at read). bits 0-7 = WalkZ N..NW (predicted via
|
||
/// WalkMask), bits 8-15 = SwimZ N..NW (predicted via WetMask).
|
||
/// byte WalkMask[256]
|
||
/// byte WetMask[256]
|
||
/// sbyte SourceZ[256]
|
||
/// // For each d in 0..15 with ZArrayMask bit d set, in N,NE,E,SE,S,SW,W,NW order
|
||
/// // (walk arrays first, then swim):
|
||
/// sbyte residual_d[256] reconstruct: Z_d[c] = (mask bit set ? SourceZ[c] : 0) + residual_d[c]
|
||
/// // Swim layer trailer — only when HasSwimLayer == 1 (chunks containing shore cells):
|
||
/// sbyte SwimSourceZ[256] (NoSwimLayerCell sentinel = sbyte.MinValue)
|
||
/// byte SwimMask[256] (per-cell swim mask baked at SwimSourceZ)
|
||
/// sbyte SwimZN_Layer[256]..SwimZNW_Layer[256] (8 arrays, dest-Z at swim perspective)
|
||
/// // Strata trailer — only when HasStrata == 1:
|
||
/// u16 StrataOffsetByCell[256] (NoStrata sentinel = 0xFFFF)
|
||
/// u32 StrataDataLength
|
||
/// byte StrataData[StrataDataLength]
|
||
/// For each multi-Z cell: u8 stratumCount, then stratumCount × Stratum (19 bytes):
|
||
/// sbyte zCenter
|
||
/// byte walkMask, wetMask
|
||
/// sbyte walkZ_N..NW (8)
|
||
/// sbyte swimZ_N..NW (8)
|
||
///
|
||
/// Index trailer (20 × ChunkCount bytes):
|
||
/// For each chunk: { u64 chunkKey, u64 fileOffset, u32 recordLength }
|
||
///
|
||
/// Per-chunk fixed portion (Kind + flags + ZArrayMask + WalkMask + WetMask + SourceZ):
|
||
/// ~783 bytes; each present base Z array adds 256 bytes (0..16 present, so up to ~4 KB).
|
||
/// A fully-flat Full chunk stores no residual blocks. Strata trailer: 516 + N × ~30 bytes
|
||
/// for a chunk with N multi-Z cells averaging ~2 strata each. LRU bookkeeping
|
||
/// (LastTouchedTicks) is intentionally not persisted.
|
||
///
|
||
/// Files with version < <see cref="MinSupportedVersion"/> are silently rejected
|
||
/// at open time (treated as missing) and overwritten on the next save.
|
||
/// </summary>
|
||
internal static class StepCacheFile
|
||
{
|
||
public const uint Magic = 0x42575300; // 'SWB\0'
|
||
public const uint FormatVersion = 7;
|
||
|
||
/// <summary>
|
||
/// Lowest format version this binary can load. Files below this version are treated as
|
||
/// missing (silently rejected) — a subsequent SaveToFile / BakeMap overwrites them with
|
||
/// the current FormatVersion. Bumped to 3 when the swim layer landed (v2 had no swim
|
||
/// layer). Bumped to 4 when the baker switched to clearance-aware standable-surface
|
||
/// strata: v3 bakes anchored every cell at the land average and so missed walkable
|
||
/// static-over-land surfaces (sewer/dungeon walkways, bridges, upper building floors),
|
||
/// producing ~98% source-Z fallthroughs on those routes. The on-disk layout is
|
||
/// unchanged; only the strata population differs, so the bump exists purely to force a
|
||
/// one-time re-bake of stale v3 files on first boot under the new binary. Bumped to 5 for
|
||
/// uniform-chunk elision: each record now begins with a Kind byte (0 = Full, 2 = Uniform);
|
||
/// a fully-uniform chunk (no strata, no swim layer, all 19 base arrays constant) stores
|
||
/// one cell's worth of data (~28 bytes) instead of the full record. Bumped to 6 for
|
||
/// predictive-Z residuals: each base directional Z array is stored as a masked residual
|
||
/// against SourceZ (a ZArrayMask u16 flags which arrays are present); arrays matching their
|
||
/// prediction are omitted and synthesized at read. v5 files are rejected and re-baked once.
|
||
/// Bumped to 7 for per-chunk compression: each chunk record is libdeflate-compressed
|
||
/// independently (random access preserved) behind a u32 uncompressed-length prefix; tiny
|
||
/// records that do not shrink are stored raw. v6 files are rejected and re-baked once.
|
||
/// </summary>
|
||
public const uint MinSupportedVersion = 7;
|
||
|
||
// Per-chunk record discriminator (first byte after BuiltMultisVersion). 1 is reserved.
|
||
private const byte KindFull = 0;
|
||
private const byte KindUniform = 2;
|
||
|
||
private const int HeaderSize =
|
||
sizeof(uint) // Magic
|
||
+ sizeof(uint) // Version
|
||
+ sizeof(uint) // MapId
|
||
+ sizeof(ulong) // Fingerprint
|
||
+ sizeof(ulong) // BakeTimestamp
|
||
+ sizeof(uint) // ChunkCount
|
||
+ sizeof(ulong); // IndexOffset
|
||
|
||
// Index entry: chunkKey + fileOffset + recordLength. Bumped to include length when
|
||
// strata made chunk records variable-size; the lazy reader uses length to do a
|
||
// single bulk read per chunk without consulting the next offset.
|
||
private const int IndexEntryBytes = sizeof(ulong) + sizeof(ulong) + sizeof(uint);
|
||
|
||
/// <summary>Fixed-size portion of a chunk record (everything except the optional strata + swim trailers).</summary>
|
||
private const int BytesPerChunkBase =
|
||
sizeof(ushort) + sizeof(ushort) + sizeof(uint)
|
||
+ sizeof(byte) + sizeof(byte) + sizeof(byte) // Kind + HasStrata + HasSwimLayer
|
||
+ sizeof(ushort) // ZArrayMask
|
||
+ StepChunk.CellsPerChunk // WalkMask
|
||
+ StepChunk.CellsPerChunk // WetMask
|
||
+ StepChunk.CellsPerChunk // SourceZ
|
||
+ 8 * StepChunk.CellsPerChunk // WalkZ[8]
|
||
+ 8 * StepChunk.CellsPerChunk; // SwimZ[8]
|
||
|
||
/// <summary>Swim-layer trailer overhead when present: per-cell SourceZ + Mask + 8×Z arrays.</summary>
|
||
private const int SwimLayerOverhead = 10 * StepChunk.CellsPerChunk;
|
||
|
||
/// <summary>Strata trailer overhead when present: 256×u16 offset table + u32 data length.</summary>
|
||
private const int StrataTrailerOverhead = StepChunk.CellsPerChunk * sizeof(ushort) + sizeof(uint);
|
||
|
||
/// <summary>
|
||
/// Byte offset of the IndexOffset u64 within the header
|
||
/// (Magic+Version+MapId+Fingerprint+BakeTimestamp+ChunkCount = 32). Patched after chunks land.
|
||
/// </summary>
|
||
private const int IndexOffsetFieldPosition = 32;
|
||
|
||
public delegate bool ChunkEnumerator(out int chunkX, out int chunkY, out StepChunk chunk);
|
||
|
||
/// <summary>
|
||
/// Peek at a .swb file's Fingerprint field (header byte offset 12) without
|
||
/// reading any chunk data. Returns false on missing file, bad magic, or wrong
|
||
/// version. Cheap — reads 20 bytes total.
|
||
/// </summary>
|
||
public static bool TryReadFingerprint(string path, out ulong fingerprint)
|
||
{
|
||
fingerprint = 0;
|
||
if (!File.Exists(path))
|
||
{
|
||
return false;
|
||
}
|
||
|
||
try
|
||
{
|
||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
|
||
Span<byte> buf = stackalloc byte[20];
|
||
if (stream.Read(buf) < 20)
|
||
{
|
||
return false;
|
||
}
|
||
if (BinaryPrimitives.ReadUInt32LittleEndian(buf) != Magic)
|
||
{
|
||
return false;
|
||
}
|
||
var version = BinaryPrimitives.ReadUInt32LittleEndian(buf[4..]);
|
||
if (version < MinSupportedVersion || version > FormatVersion)
|
||
{
|
||
return false;
|
||
}
|
||
// mapId is at buf[8..12], we skip; hash is at buf[12..20].
|
||
fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(buf[12..]);
|
||
return true;
|
||
}
|
||
catch
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Combined XxHash3 fingerprint over (1) the loaded TileData flag tables and (2) the
|
||
/// per-map .mul / .uop file contents (via <see cref="TileMatrix.MapFilesFingerprint"/>).
|
||
/// Bake files carry this hash so a load can refuse to populate the cache when EITHER
|
||
/// tile flags shifted (client patch) OR the map data was rewritten (CentredSharp /
|
||
/// UOFiddler edit). The .mul format has no built-in CRC; this is the only way to
|
||
/// detect those mutations.
|
||
/// </summary>
|
||
public static ulong ComputeFingerprint(int mapId)
|
||
{
|
||
var hasher = HashUtility.CreateXxHash3();
|
||
|
||
// TileData flag tables — same projection trick as before: just the Flags ulong
|
||
// from each entry, written little-endian into a contiguous byte buffer. The
|
||
// struct itself has a string Name (reference) whose object identity isn't
|
||
// stable across runs, so MemoryMarshal.Cast over the whole struct would drift.
|
||
var landTable = TileData.LandTable;
|
||
var itemTable = TileData.ItemTable;
|
||
var bytes = new byte[(landTable.Length + itemTable.Length) * sizeof(ulong)];
|
||
var span = bytes.AsSpan();
|
||
|
||
for (var i = 0; i < landTable.Length; i++)
|
||
{
|
||
BinaryPrimitives.WriteUInt64LittleEndian(span[(i * 8)..], (ulong)landTable[i].Flags);
|
||
}
|
||
var itemOffset = landTable.Length * 8;
|
||
for (var i = 0; i < itemTable.Length; i++)
|
||
{
|
||
BinaryPrimitives.WriteUInt64LittleEndian(span[(itemOffset + i * 8)..], (ulong)itemTable[i].Flags);
|
||
}
|
||
hasher.Append(bytes);
|
||
|
||
// Map files (mapX.mul / .uop, staidxX.mul, staticsX.mul). TileMatrix already
|
||
// streamed them through XxHash3 once at construction; mix the result in.
|
||
var map = Map.Maps[mapId];
|
||
if (map != null && map != Map.Internal && map.Tiles != null)
|
||
{
|
||
Span<byte> mapHashBytes = stackalloc byte[sizeof(ulong)];
|
||
BinaryPrimitives.WriteUInt64LittleEndian(mapHashBytes, map.Tiles.MapFilesFingerprint);
|
||
hasher.Append(mapHashBytes);
|
||
}
|
||
|
||
return hasher.GetCurrentHashAsUInt64();
|
||
}
|
||
|
||
/// <summary>
|
||
/// Writes the file: header (with placeholder IndexOffset) → chunks (offsets recorded)
|
||
/// → index trailer → patches the header IndexOffset. <paramref name="chunkCount"/> must
|
||
/// equal the actual number of chunks <paramref name="next"/> will yield.
|
||
/// </summary>
|
||
public static void Write(string path, uint mapId, uint chunkCount, ChunkEnumerator next)
|
||
{
|
||
Directory.CreateDirectory(Path.GetDirectoryName(path) ?? ".");
|
||
|
||
// Initial estimate: base record + a modest strata budget per chunk. Coastline
|
||
// chunks add another ~2.5 KB (swim layer) but they're a small fraction of any
|
||
// map; the writer grows on overflow so under-estimating just causes a few
|
||
// realloc/copy cycles during the bake — not a correctness issue.
|
||
var capacity = HeaderSize
|
||
+ (BytesPerChunkBase + 256) * (int)chunkCount
|
||
+ IndexEntryBytes * (int)chunkCount;
|
||
var buffer = new byte[capacity];
|
||
var w = new BufferWriter(buffer, prefixStr: false);
|
||
|
||
w.Write(Magic);
|
||
w.Write(FormatVersion);
|
||
w.Write(mapId);
|
||
w.Write(ComputeFingerprint((int)mapId));
|
||
w.Write((ulong)DateTime.UtcNow.Ticks);
|
||
w.Write(chunkCount);
|
||
w.Write(0UL); // IndexOffset placeholder, patched after chunks
|
||
|
||
// Per-chunk compression: each record is built uncompressed into `recordScratch`, then
|
||
// libdeflate-compressed into `compScratch` and framed as [u32 uncompressedLen][payload].
|
||
// Reuse the cached per-thread VeryHigh compressor (construction allocates native state).
|
||
// libdeflate is not thread-safe, but bake runs single-threaded. VeryHigh is the best-ratio
|
||
// level and its slow compression is irrelevant offline (decompression is what's hot, ~1.8us).
|
||
var packer = Deflate.Maximum;
|
||
var recordScratch = new byte[BytesPerChunkBase + 1024];
|
||
var compScratch = new byte[packer.MaxPackSize(recordScratch.Length)];
|
||
|
||
var indexEntries = new (ulong key, ulong offset, uint length)[chunkCount];
|
||
var written = 0u;
|
||
while (next(out var chunkX, out var chunkY, out var chunk))
|
||
{
|
||
if (written >= chunkCount)
|
||
{
|
||
throw new InvalidOperationException(
|
||
$"StepCacheFile.Write: enumerator yielded more than the declared {chunkCount} chunks"
|
||
);
|
||
}
|
||
var chunkOffset = (ulong)w.Position;
|
||
WriteChunk(w, chunkX, chunkY, chunk, packer, ref recordScratch, ref compScratch);
|
||
var chunkLength = (uint)((ulong)w.Position - chunkOffset);
|
||
indexEntries[written] = (PackChunkKey(chunkX, chunkY), chunkOffset, chunkLength);
|
||
written++;
|
||
}
|
||
|
||
if (written != chunkCount)
|
||
{
|
||
throw new InvalidOperationException(
|
||
$"StepCacheFile.Write: declared {chunkCount} chunks but enumerator yielded {written}"
|
||
);
|
||
}
|
||
|
||
var indexOffset = (ulong)w.Position;
|
||
for (var i = 0u; i < chunkCount; i++)
|
||
{
|
||
w.Write(indexEntries[i].key);
|
||
w.Write(indexEntries[i].offset);
|
||
w.Write(indexEntries[i].length);
|
||
}
|
||
|
||
// Patch IndexOffset on the writer's current backing buffer (BufferWriter may
|
||
// have grown during chunk writes; the original `buffer` ref is stale after grow).
|
||
var liveBuffer = w.Buffer;
|
||
BinaryPrimitives.WriteUInt64LittleEndian(liveBuffer.AsSpan(IndexOffsetFieldPosition, 8), indexOffset);
|
||
|
||
var totalBytes = (int)w.Position;
|
||
File.WriteAllBytes(path, liveBuffer.AsSpan(0, totalBytes).ToArray());
|
||
}
|
||
|
||
/// <summary>
|
||
/// Opens a .swb file and reads only its header + chunk-offset index. Returns null on
|
||
/// missing file, magic / version mismatch, or Fingerprint mismatch (a stale bake
|
||
/// against a freshly patched client). Callers own disposal of the returned reader.
|
||
/// </summary>
|
||
public static LazyReader OpenForLazy(string path)
|
||
{
|
||
if (!File.Exists(path))
|
||
{
|
||
return null;
|
||
}
|
||
|
||
FileStream stream = null;
|
||
try
|
||
{
|
||
stream = new FileStream(
|
||
path,
|
||
FileMode.Open,
|
||
FileAccess.Read,
|
||
FileShare.Read | FileShare.Delete
|
||
);
|
||
|
||
Span<byte> headerBuf = stackalloc byte[HeaderSize];
|
||
if (stream.Read(headerBuf) != HeaderSize)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
var magic = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf);
|
||
if (magic != Magic)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
var version = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[4..]);
|
||
if (version < MinSupportedVersion || version > FormatVersion)
|
||
{
|
||
// Below the minimum supported version: treat as missing. Older files
|
||
// get silently overwritten on the next SaveToFile / BakeMap.
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
var mapId = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[8..]);
|
||
var fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[12..]);
|
||
var bakeTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[20..]);
|
||
var chunkCount = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[28..]);
|
||
var indexOffset = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[32..]);
|
||
|
||
if (fingerprint != ComputeFingerprint((int)mapId))
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
// Read the chunk-offset index in one shot.
|
||
var indexBytes = (int)chunkCount * IndexEntryBytes;
|
||
var indexBuf = new byte[indexBytes];
|
||
stream.Position = (long)indexOffset;
|
||
if (stream.Read(indexBuf, 0, indexBytes) != indexBytes)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
var offsets = new Dictionary<ulong, (ulong offset, uint length)>((int)chunkCount);
|
||
for (var i = 0; i < chunkCount; i++)
|
||
{
|
||
var entry = indexBuf.AsSpan(i * IndexEntryBytes);
|
||
var key = BinaryPrimitives.ReadUInt64LittleEndian(entry);
|
||
var off = BinaryPrimitives.ReadUInt64LittleEndian(entry[8..]);
|
||
var len = BinaryPrimitives.ReadUInt32LittleEndian(entry[16..]);
|
||
offsets[key] = (off, len);
|
||
}
|
||
|
||
return new LazyReader(stream, mapId, fingerprint, bakeTimestamp, chunkCount, offsets);
|
||
}
|
||
catch
|
||
{
|
||
stream?.Dispose();
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private static ulong PackChunkKey(int chunkX, int chunkY) => ((ulong)(uint)chunkX << 32) | (uint)chunkY;
|
||
|
||
/// <summary>
|
||
/// Predicted directional-Z for one cell/direction: the cell's own SourceZ when the
|
||
/// direction is walkable/wet (mask bit set), else 0 — matching the baker, which leaves
|
||
/// non-walkable directional slots at their zero-initialized default
|
||
/// (StepProbe.ComputeMaskAt clears walkZs/swimZs and writes only on a successful step).
|
||
/// </summary>
|
||
internal static sbyte Predict(byte dirMaskByte, int bit, sbyte sourceZ) =>
|
||
(dirMaskByte >> bit & 1) != 0 ? sourceZ : (sbyte)0;
|
||
|
||
/// <summary>
|
||
/// Residual of an absolute directional-Z against its prediction. Unchecked two's-complement
|
||
/// so the transform is byte-exact for ALL sbyte inputs (no value-range constraint).
|
||
/// </summary>
|
||
internal static sbyte EncodeResidual(sbyte z, sbyte predict) => unchecked((sbyte)(z - predict));
|
||
|
||
/// <summary>Inverse of <see cref="EncodeResidual"/>: absolute directional-Z = predict + residual.</summary>
|
||
internal static sbyte DecodeZ(sbyte predict, sbyte residual) => unchecked((sbyte)(predict + residual));
|
||
|
||
/// <summary>
|
||
/// The 16 base directional-Z arrays in canonical order: walk N..NW (indices 0-7),
|
||
/// then swim N..NW (8-15). Index d uses WalkMask (d < 8) or WetMask (d >= 8)
|
||
/// with direction bit (d & 7). Allocates a 16-slot reference array (bake/read time only).
|
||
/// </summary>
|
||
private static sbyte[][] GetBaseZArrays(StepChunk c) => new[]
|
||
{
|
||
c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW,
|
||
c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW,
|
||
};
|
||
|
||
/// <summary>
|
||
/// Builds the uncompressed v6 record for one chunk into <paramref name="w"/>, libdeflate-
|
||
/// compresses it, and writes it framed as [u32 uncompressedLen][payload]. The payload is the
|
||
/// compressed bytes, or — when compression does not shrink the record (tiny Uniform records) —
|
||
/// the raw record itself; the reader distinguishes the two by payload length vs uncompressedLen.
|
||
/// </summary>
|
||
private static void WriteChunk(
|
||
BufferWriter w, int chunkX, int chunkY, StepChunk chunk,
|
||
LibDeflateBinding packer, ref byte[] recordScratch, ref byte[] compScratch
|
||
)
|
||
{
|
||
var rw = new BufferWriter(recordScratch, prefixStr: false);
|
||
BuildRecord(rw, chunkX, chunkY, chunk);
|
||
recordScratch = rw.Buffer; // may have grown; keep the larger buffer for reuse
|
||
var recordLen = (int)rw.Position;
|
||
|
||
var bound = packer.MaxPackSize(recordLen);
|
||
if (compScratch.Length < bound)
|
||
{
|
||
compScratch = new byte[bound];
|
||
}
|
||
|
||
var compLen = packer.Pack(compScratch, recordScratch.AsSpan(0, recordLen));
|
||
|
||
w.Write((uint)recordLen);
|
||
if (compLen > 0 && compLen < recordLen)
|
||
{
|
||
w.Write(compScratch.AsSpan(0, compLen));
|
||
}
|
||
else
|
||
{
|
||
// Incompressible (or expanded): store the record raw. The reader detects this when
|
||
// the on-disk payload length equals the uncompressed length.
|
||
w.Write(recordScratch.AsSpan(0, recordLen));
|
||
}
|
||
}
|
||
|
||
private static void BuildRecord(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
|
||
{
|
||
w.Write((ushort)chunkX);
|
||
w.Write((ushort)chunkY);
|
||
w.Write((uint)chunk.BuiltMultisVersion);
|
||
|
||
// Kind: 0 = Full, 2 = Uniform. A uniform chunk (no strata, no swim layer, all 19 base
|
||
// arrays constant) stores one cell's worth of data (~28-byte record total).
|
||
if (chunk.IsUniform())
|
||
{
|
||
w.Write(KindUniform);
|
||
w.Write(chunk.WalkMask[0]);
|
||
w.Write(chunk.WetMask[0]);
|
||
w.Write((byte)chunk.SourceZ[0]);
|
||
w.Write((byte)chunk.WalkZN[0]);
|
||
w.Write((byte)chunk.WalkZNE[0]);
|
||
w.Write((byte)chunk.WalkZE[0]);
|
||
w.Write((byte)chunk.WalkZSE[0]);
|
||
w.Write((byte)chunk.WalkZS[0]);
|
||
w.Write((byte)chunk.WalkZSW[0]);
|
||
w.Write((byte)chunk.WalkZW[0]);
|
||
w.Write((byte)chunk.WalkZNW[0]);
|
||
w.Write((byte)chunk.SwimZN[0]);
|
||
w.Write((byte)chunk.SwimZNE[0]);
|
||
w.Write((byte)chunk.SwimZE[0]);
|
||
w.Write((byte)chunk.SwimZSE[0]);
|
||
w.Write((byte)chunk.SwimZS[0]);
|
||
w.Write((byte)chunk.SwimZSW[0]);
|
||
w.Write((byte)chunk.SwimZW[0]);
|
||
w.Write((byte)chunk.SwimZNW[0]);
|
||
return;
|
||
}
|
||
|
||
w.Write(KindFull); // Full
|
||
|
||
var strataOffsetByCell = chunk.GetStrataOffsetByCellForSerialization();
|
||
var strataData = chunk.GetStrataDataForSerialization();
|
||
var hasStrata = strataOffsetByCell != null;
|
||
var hasSwimLayer = chunk.HasSwimLayer;
|
||
w.Write((byte)(hasStrata ? 1 : 0));
|
||
w.Write((byte)(hasSwimLayer ? 1 : 0));
|
||
|
||
// Predictive-Z: each base directional Z array is stored as a masked residual against
|
||
// SourceZ. Bit d of ZArrayMask is set only when array d differs from its prediction
|
||
// somewhere; cleared arrays are omitted and rebuilt from mask+SourceZ at read.
|
||
var zArrays = GetBaseZArrays(chunk);
|
||
ushort zArrayMask = 0;
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
var z = zArrays[d];
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
if (z[cell] != Predict(dirMask[cell], bit, chunk.SourceZ[cell]))
|
||
{
|
||
zArrayMask |= (ushort)(1 << d);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
w.Write(zArrayMask);
|
||
|
||
w.Write(chunk.WalkMask);
|
||
w.Write(chunk.WetMask);
|
||
WriteSBytes(w, chunk.SourceZ);
|
||
|
||
Span<sbyte> residual = stackalloc sbyte[StepChunk.CellsPerChunk];
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
if ((zArrayMask >> d & 1) == 0)
|
||
{
|
||
continue;
|
||
}
|
||
var z = zArrays[d];
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
residual[cell] = EncodeResidual(z[cell], Predict(dirMask[cell], bit, chunk.SourceZ[cell]));
|
||
}
|
||
w.Write(MemoryMarshal.Cast<sbyte, byte>(residual));
|
||
}
|
||
|
||
if (hasSwimLayer)
|
||
{
|
||
WriteSBytes(w, chunk.SwimSourceZ);
|
||
w.Write(chunk.SwimMask);
|
||
WriteSBytes(w, chunk.SwimZN_Layer);
|
||
WriteSBytes(w, chunk.SwimZNE_Layer);
|
||
WriteSBytes(w, chunk.SwimZE_Layer);
|
||
WriteSBytes(w, chunk.SwimZSE_Layer);
|
||
WriteSBytes(w, chunk.SwimZS_Layer);
|
||
WriteSBytes(w, chunk.SwimZSW_Layer);
|
||
WriteSBytes(w, chunk.SwimZW_Layer);
|
||
WriteSBytes(w, chunk.SwimZNW_Layer);
|
||
}
|
||
|
||
if (hasStrata)
|
||
{
|
||
// 256 × u16 offsets, then u32 length-prefixed strata byte array.
|
||
for (var i = 0; i < StepChunk.CellsPerChunk; i++)
|
||
{
|
||
w.Write(strataOffsetByCell[i]);
|
||
}
|
||
var dataLen = (uint)(strataData?.Length ?? 0);
|
||
w.Write(dataLen);
|
||
if (dataLen > 0)
|
||
{
|
||
w.Write(strataData);
|
||
}
|
||
}
|
||
}
|
||
|
||
private static StepChunk ReadChunk(byte[] buffer)
|
||
{
|
||
var r = new BufferReader(buffer);
|
||
// Skip ChunkX + ChunkY (already known via the index lookup).
|
||
r.ReadUShort();
|
||
r.ReadUShort();
|
||
var multisVersion = (int)r.ReadUInt();
|
||
var kind = r.ReadByte();
|
||
|
||
var chunk = new StepChunk { BuiltMultisVersion = multisVersion };
|
||
|
||
if (kind == KindUniform) // Uniform — one cell's worth of the 19 base arrays, fill all 256 cells.
|
||
{
|
||
Array.Fill(chunk.WalkMask, r.ReadByte());
|
||
Array.Fill(chunk.WetMask, r.ReadByte());
|
||
Array.Fill(chunk.SourceZ, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZN, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZNE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZSE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZS, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZSW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZNW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZN, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZNE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZSE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZS, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZSW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZNW, (sbyte)r.ReadByte());
|
||
return chunk;
|
||
}
|
||
|
||
var hasStrata = r.ReadByte() != 0;
|
||
var hasSwimLayer = r.ReadByte() != 0;
|
||
var zArrayMask = r.ReadUShort();
|
||
|
||
r.Read(chunk.WalkMask);
|
||
r.Read(chunk.WetMask);
|
||
ReadSBytes(r, chunk.SourceZ);
|
||
|
||
// Predictive-Z reconstruction: present arrays carry residuals (z = predict + residual);
|
||
// absent arrays are synthesized from mask+SourceZ (z = predict, residual implicitly 0).
|
||
var zArrays = GetBaseZArrays(chunk);
|
||
Span<sbyte> residual = stackalloc sbyte[StepChunk.CellsPerChunk];
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
var z = zArrays[d];
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
if ((zArrayMask >> d & 1) != 0)
|
||
{
|
||
r.Read(MemoryMarshal.Cast<sbyte, byte>(residual));
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
z[cell] = DecodeZ(Predict(dirMask[cell], bit, chunk.SourceZ[cell]), residual[cell]);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
z[cell] = Predict(dirMask[cell], bit, chunk.SourceZ[cell]);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (hasSwimLayer)
|
||
{
|
||
chunk.AllocateSwimLayer();
|
||
ReadSBytes(r, chunk.SwimSourceZ);
|
||
r.Read(chunk.SwimMask);
|
||
ReadSBytes(r, chunk.SwimZN_Layer);
|
||
ReadSBytes(r, chunk.SwimZNE_Layer);
|
||
ReadSBytes(r, chunk.SwimZE_Layer);
|
||
ReadSBytes(r, chunk.SwimZSE_Layer);
|
||
ReadSBytes(r, chunk.SwimZS_Layer);
|
||
ReadSBytes(r, chunk.SwimZSW_Layer);
|
||
ReadSBytes(r, chunk.SwimZW_Layer);
|
||
ReadSBytes(r, chunk.SwimZNW_Layer);
|
||
}
|
||
|
||
if (hasStrata)
|
||
{
|
||
var offsets = new ushort[StepChunk.CellsPerChunk];
|
||
for (var i = 0; i < offsets.Length; i++)
|
||
{
|
||
offsets[i] = r.ReadUShort();
|
||
}
|
||
var dataLen = (int)r.ReadUInt();
|
||
var data = new byte[dataLen];
|
||
if (dataLen > 0)
|
||
{
|
||
r.Read(data);
|
||
}
|
||
chunk.SetStrata(offsets, data);
|
||
}
|
||
|
||
return chunk;
|
||
}
|
||
|
||
private static void WriteSBytes(BufferWriter w, sbyte[] arr) =>
|
||
w.Write(MemoryMarshal.Cast<sbyte, byte>(arr.AsSpan()));
|
||
|
||
private static void ReadSBytes(BufferReader r, sbyte[] arr) =>
|
||
r.Read(MemoryMarshal.Cast<sbyte, byte>(arr.AsSpan()));
|
||
|
||
/// <summary>
|
||
/// Open handle on a .swb file. Holds the FileStream + chunk-offset index. Chunks are
|
||
/// fetched on demand via <see cref="TryReadChunk"/>; only the records actually queried
|
||
/// are ever materialized. Dispose releases the underlying stream.
|
||
/// </summary>
|
||
internal sealed class LazyReader : IDisposable
|
||
{
|
||
private FileStream _stream;
|
||
private readonly Dictionary<ulong, (ulong offset, uint length)> _offsets;
|
||
private byte[] _buffer; // raw on-disk record: [u32 uncompressedLen][payload]
|
||
private byte[] _bodyBuffer; // decompressed v6 record, parsed by ReadChunk
|
||
|
||
public uint MapId { get; }
|
||
public ulong Fingerprint { get; }
|
||
public ulong BakeTimestamp { get; }
|
||
public uint ChunkCount { get; }
|
||
public int IndexedChunkCount => _offsets.Count;
|
||
|
||
public bool Has(int chunkX, int chunkY) => _offsets.ContainsKey(PackChunkKey(chunkX, chunkY));
|
||
|
||
/// <summary>
|
||
/// Enumerates every (chunkX, chunkY) coordinate the file holds. Used by
|
||
/// <see cref="StepCache"/> when preload is enabled to materialize all chunks
|
||
/// upfront instead of on first query.
|
||
/// </summary>
|
||
public IEnumerable<(int chunkX, int chunkY)> EnumerateChunkCoords()
|
||
{
|
||
foreach (var key in _offsets.Keys)
|
||
{
|
||
yield return ((int)(key >> 32), (int)(key & 0xFFFFFFFF));
|
||
}
|
||
}
|
||
|
||
internal LazyReader(
|
||
FileStream stream, uint mapId, ulong fingerprint, ulong bakeTimestamp,
|
||
uint chunkCount, Dictionary<ulong, (ulong offset, uint length)> offsets
|
||
)
|
||
{
|
||
_stream = stream;
|
||
MapId = mapId;
|
||
Fingerprint = fingerprint;
|
||
BakeTimestamp = bakeTimestamp;
|
||
ChunkCount = chunkCount;
|
||
_offsets = offsets;
|
||
_buffer = new byte[BytesPerChunkBase];
|
||
_bodyBuffer = new byte[BytesPerChunkBase];
|
||
}
|
||
|
||
/// <summary>
|
||
/// Returns the chunk record at (<paramref name="chunkX"/>, <paramref name="chunkY"/>)
|
||
/// from the file, or null if the file doesn't contain it. Single seek + bulk read,
|
||
/// sized exactly to the chunk's recorded length (which varies with strata size).
|
||
/// </summary>
|
||
public StepChunk TryReadChunk(int chunkX, int chunkY)
|
||
{
|
||
if (_stream == null)
|
||
{
|
||
return null;
|
||
}
|
||
|
||
var key = PackChunkKey(chunkX, chunkY);
|
||
if (!_offsets.TryGetValue(key, out var entry))
|
||
{
|
||
return null;
|
||
}
|
||
|
||
// Grow the on-disk scratch buffer if this chunk's record is larger than what we have.
|
||
if (entry.length > _buffer.Length)
|
||
{
|
||
_buffer = new byte[entry.length];
|
||
}
|
||
|
||
_stream.Position = (long)entry.offset;
|
||
var read = _stream.Read(_buffer, 0, (int)entry.length);
|
||
if (read < (int)entry.length || entry.length < sizeof(uint))
|
||
{
|
||
return null;
|
||
}
|
||
|
||
// Frame: [u32 uncompressedLen][payload]. payload is libdeflate-compressed, unless its
|
||
// length equals uncompressedLen, in which case it was stored raw (incompressible).
|
||
var uncompressedLen = (int)BinaryPrimitives.ReadUInt32LittleEndian(_buffer);
|
||
var payloadLen = (int)entry.length - sizeof(uint);
|
||
if (_bodyBuffer.Length < uncompressedLen)
|
||
{
|
||
_bodyBuffer = new byte[uncompressedLen];
|
||
}
|
||
|
||
if (payloadLen == uncompressedLen)
|
||
{
|
||
Array.Copy(_buffer, sizeof(uint), _bodyBuffer, 0, uncompressedLen);
|
||
}
|
||
else
|
||
{
|
||
// Decompression is level-independent, so reuse the shared per-thread binding
|
||
// rather than allocating a native decompressor per reader. The cache is read on
|
||
// the single game thread; libdeflate's non-thread-safety is satisfied by ThreadStatic.
|
||
var result = Deflate.Standard.Unpack(
|
||
_bodyBuffer.AsSpan(0, uncompressedLen),
|
||
_buffer.AsSpan(sizeof(uint), payloadLen),
|
||
out var produced
|
||
);
|
||
if (result != LibDeflateResult.Success || produced != uncompressedLen)
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
return ReadChunk(_bodyBuffer);
|
||
}
|
||
|
||
public void Dispose()
|
||
{
|
||
_stream?.Dispose();
|
||
_stream = null;
|
||
_buffer = null;
|
||
_bodyBuffer = null;
|
||
}
|
||
}
|
||
}
|