94537f83f8
## Summary Phase #2 of the `.swb` step-cache size-reduction roadmap (after #2465, v5 uniform elision). Stores the 16 base directional Z arrays as masked residuals against each cell's own SourceZ and omits any array that matches its prediction. Lossless, byte-identical reconstruction. Trammel: 231.9 MB → 124.7 MB (−46%). ## Details - Predictor: `predict = mask bit ? SourceZ : 0` (matches the baker's 0 on unwalkable directions); residual `Z − predict` via unchecked two's-complement (byte-exact for all inputs); reconstruct `Z = predict + residual`. - A `u16 ZArrayMask` flags which of the 16 base arrays differ from prediction; matching arrays are omitted and synthesized from mask + SourceZ at read. - Serializer-layer only: StepChunk, the cache, the algorithm, and the baker are unchanged. - Format v6; v5 files rejected and re-baked once. ## Tests 21 v6 unit tests; full pathfinding suite green; Release build clean.
721 lines
30 KiB
C#
721 lines
30 KiB
C#
using System;
|
||
using System.Buffers.Binary;
|
||
using System.Collections.Generic;
|
||
using System.IO;
|
||
using System.Runtime.InteropServices;
|
||
|
||
namespace Server.Engines.Pathing.Cache;
|
||
|
||
/// <summary>
|
||
/// Binary serializer + lazy reader for the step cache. Persists chunk records to disk
|
||
/// so a server warm-starts without paying chunk-build cost on the first pathfind through
|
||
/// a region. Lazy: opening a file reads only the header + chunk-offset index (~few KB
|
||
/// for tens of thousands of chunks), then individual chunks are seeked + deserialized
|
||
/// only when the cache asks for them. RAM stays bounded by MaxResidentChunks regardless
|
||
/// of file size.
|
||
///
|
||
/// File layout v6 (little-endian, BufferWriter / BufferReader convention):
|
||
///
|
||
/// Header (48 bytes):
|
||
/// u32 Magic = 0x42575300 ('SWB\0')
|
||
/// u32 Version = current FormatVersion (6)
|
||
/// u32 MapId
|
||
/// u64 Fingerprint XxHash3 over (1) LandTable + ItemTable flags AND (2) the
|
||
/// on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul.
|
||
/// Rejects a load when EITHER tile flags shifted (client patch)
|
||
/// OR the map data was rewritten (CentredSharp / UOFiddler edit).
|
||
/// The .mul format has no built-in CRC; this is the only way
|
||
/// to detect those mutations.
|
||
/// u64 BakeTimestamp DateTime.UtcNow.Ticks at write time (informational).
|
||
/// u32 ChunkCount
|
||
/// u64 IndexOffset File position where the chunk index begins.
|
||
///
|
||
/// Per chunk (ChunkCount times, variable size):
|
||
/// u16 ChunkX
|
||
/// u16 ChunkY
|
||
/// u32 BuiltMultisVersion
|
||
/// u8 Kind 0 = Full; 2 = Uniform
|
||
/// // Uniform (Kind == 2): ~28-byte record — all 256 cells share these single values:
|
||
/// byte walkMask, wetMask; sbyte sourceZ; sbyte walkZ_N..NW (8); sbyte swimZ_N..NW (8)
|
||
/// // Full (Kind == 0) body:
|
||
/// u8 HasStrata 0 = single-Z chunk (no strata trailer); 1 = strata trailer follows
|
||
/// u8 HasSwimLayer 0 = no shore cells (no swim trailer); 1 = swim trailer follows
|
||
/// u16 ZArrayMask bit d set => base directional Z array d is present below as a
|
||
/// residual[256] block; cleared => array equals its prediction and is
|
||
/// omitted (synthesized at read). bits 0-7 = WalkZ N..NW (predicted via
|
||
/// WalkMask), bits 8-15 = SwimZ N..NW (predicted via WetMask).
|
||
/// byte WalkMask[256]
|
||
/// byte WetMask[256]
|
||
/// sbyte SourceZ[256]
|
||
/// // For each d in 0..15 with ZArrayMask bit d set, in N,NE,E,SE,S,SW,W,NW order
|
||
/// // (walk arrays first, then swim):
|
||
/// sbyte residual_d[256] reconstruct: Z_d[c] = (mask bit set ? SourceZ[c] : 0) + residual_d[c]
|
||
/// // Swim layer trailer — only when HasSwimLayer == 1 (chunks containing shore cells):
|
||
/// sbyte SwimSourceZ[256] (NoSwimLayerCell sentinel = sbyte.MinValue)
|
||
/// byte SwimMask[256] (per-cell swim mask baked at SwimSourceZ)
|
||
/// sbyte SwimZN_Layer[256]..SwimZNW_Layer[256] (8 arrays, dest-Z at swim perspective)
|
||
/// // Strata trailer — only when HasStrata == 1:
|
||
/// u16 StrataOffsetByCell[256] (NoStrata sentinel = 0xFFFF)
|
||
/// u32 StrataDataLength
|
||
/// byte StrataData[StrataDataLength]
|
||
/// For each multi-Z cell: u8 stratumCount, then stratumCount × Stratum (19 bytes):
|
||
/// sbyte zCenter
|
||
/// byte walkMask, wetMask
|
||
/// sbyte walkZ_N..NW (8)
|
||
/// sbyte swimZ_N..NW (8)
|
||
///
|
||
/// Index trailer (20 × ChunkCount bytes):
|
||
/// For each chunk: { u64 chunkKey, u64 fileOffset, u32 recordLength }
|
||
///
|
||
/// Per-chunk fixed portion (Kind + flags + ZArrayMask + WalkMask + WetMask + SourceZ):
|
||
/// ~783 bytes; each present base Z array adds 256 bytes (0..16 present, so up to ~4 KB).
|
||
/// A fully-flat Full chunk stores no residual blocks. Strata trailer: 516 + N × ~30 bytes
|
||
/// for a chunk with N multi-Z cells averaging ~2 strata each. LRU bookkeeping
|
||
/// (LastTouchedTicks) is intentionally not persisted.
|
||
///
|
||
/// Files with version < <see cref="MinSupportedVersion"/> are silently rejected
|
||
/// at open time (treated as missing) and overwritten on the next save.
|
||
/// </summary>
|
||
internal static class StepCacheFile
|
||
{
|
||
public const uint Magic = 0x42575300; // 'SWB\0'
|
||
public const uint FormatVersion = 6;
|
||
|
||
/// <summary>
|
||
/// Lowest format version this binary can load. Files below this version are treated as
|
||
/// missing (silently rejected) — a subsequent SaveToFile / BakeMap overwrites them with
|
||
/// the current FormatVersion. Bumped to 3 when the swim layer landed (v2 had no swim
|
||
/// layer). Bumped to 4 when the baker switched to clearance-aware standable-surface
|
||
/// strata: v3 bakes anchored every cell at the land average and so missed walkable
|
||
/// static-over-land surfaces (sewer/dungeon walkways, bridges, upper building floors),
|
||
/// producing ~98% source-Z fallthroughs on those routes. The on-disk layout is
|
||
/// unchanged; only the strata population differs, so the bump exists purely to force a
|
||
/// one-time re-bake of stale v3 files on first boot under the new binary. Bumped to 5 for
|
||
/// uniform-chunk elision: each record now begins with a Kind byte (0 = Full, 2 = Uniform);
|
||
/// a fully-uniform chunk (no strata, no swim layer, all 19 base arrays constant) stores
|
||
/// one cell's worth of data (~28 bytes) instead of the full record. Bumped to 6 for
|
||
/// predictive-Z residuals: each base directional Z array is stored as a masked residual
|
||
/// against SourceZ (a ZArrayMask u16 flags which arrays are present); arrays matching their
|
||
/// prediction are omitted and synthesized at read. v5 files are rejected and re-baked once.
|
||
/// </summary>
|
||
public const uint MinSupportedVersion = 6;
|
||
|
||
// Per-chunk record discriminator (first byte after BuiltMultisVersion). 1 is reserved.
|
||
private const byte KindFull = 0;
|
||
private const byte KindUniform = 2;
|
||
|
||
private const int HeaderSize =
|
||
sizeof(uint) // Magic
|
||
+ sizeof(uint) // Version
|
||
+ sizeof(uint) // MapId
|
||
+ sizeof(ulong) // Fingerprint
|
||
+ sizeof(ulong) // BakeTimestamp
|
||
+ sizeof(uint) // ChunkCount
|
||
+ sizeof(ulong); // IndexOffset
|
||
|
||
// Index entry: chunkKey + fileOffset + recordLength. Bumped to include length when
|
||
// strata made chunk records variable-size; the lazy reader uses length to do a
|
||
// single bulk read per chunk without consulting the next offset.
|
||
private const int IndexEntryBytes = sizeof(ulong) + sizeof(ulong) + sizeof(uint);
|
||
|
||
/// <summary>Fixed-size portion of a chunk record (everything except the optional strata + swim trailers).</summary>
|
||
private const int BytesPerChunkBase =
|
||
sizeof(ushort) + sizeof(ushort) + sizeof(uint)
|
||
+ sizeof(byte) + sizeof(byte) + sizeof(byte) // Kind + HasStrata + HasSwimLayer
|
||
+ sizeof(ushort) // ZArrayMask
|
||
+ StepChunk.CellsPerChunk // WalkMask
|
||
+ StepChunk.CellsPerChunk // WetMask
|
||
+ StepChunk.CellsPerChunk // SourceZ
|
||
+ 8 * StepChunk.CellsPerChunk // WalkZ[8]
|
||
+ 8 * StepChunk.CellsPerChunk; // SwimZ[8]
|
||
|
||
/// <summary>Swim-layer trailer overhead when present: per-cell SourceZ + Mask + 8×Z arrays.</summary>
|
||
private const int SwimLayerOverhead = 10 * StepChunk.CellsPerChunk;
|
||
|
||
/// <summary>Strata trailer overhead when present: 256×u16 offset table + u32 data length.</summary>
|
||
private const int StrataTrailerOverhead = StepChunk.CellsPerChunk * sizeof(ushort) + sizeof(uint);
|
||
|
||
/// <summary>
|
||
/// Byte offset of the IndexOffset u64 within the header
|
||
/// (Magic+Version+MapId+Fingerprint+BakeTimestamp+ChunkCount = 32). Patched after chunks land.
|
||
/// </summary>
|
||
private const int IndexOffsetFieldPosition = 32;
|
||
|
||
public delegate bool ChunkEnumerator(out int chunkX, out int chunkY, out StepChunk chunk);
|
||
|
||
/// <summary>
|
||
/// Peek at a .swb file's Fingerprint field (header byte offset 12) without
|
||
/// reading any chunk data. Returns false on missing file, bad magic, or wrong
|
||
/// version. Cheap — reads 20 bytes total.
|
||
/// </summary>
|
||
public static bool TryReadFingerprint(string path, out ulong fingerprint)
|
||
{
|
||
fingerprint = 0;
|
||
if (!File.Exists(path))
|
||
{
|
||
return false;
|
||
}
|
||
|
||
try
|
||
{
|
||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
|
||
Span<byte> buf = stackalloc byte[20];
|
||
if (stream.Read(buf) < 20)
|
||
{
|
||
return false;
|
||
}
|
||
if (BinaryPrimitives.ReadUInt32LittleEndian(buf) != Magic)
|
||
{
|
||
return false;
|
||
}
|
||
var version = BinaryPrimitives.ReadUInt32LittleEndian(buf[4..]);
|
||
if (version < MinSupportedVersion || version > FormatVersion)
|
||
{
|
||
return false;
|
||
}
|
||
// mapId is at buf[8..12], we skip; hash is at buf[12..20].
|
||
fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(buf[12..]);
|
||
return true;
|
||
}
|
||
catch
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Combined XxHash3 fingerprint over (1) the loaded TileData flag tables and (2) the
|
||
/// per-map .mul / .uop file contents (via <see cref="TileMatrix.MapFilesFingerprint"/>).
|
||
/// Bake files carry this hash so a load can refuse to populate the cache when EITHER
|
||
/// tile flags shifted (client patch) OR the map data was rewritten (CentredSharp /
|
||
/// UOFiddler edit). The .mul format has no built-in CRC; this is the only way to
|
||
/// detect those mutations.
|
||
/// </summary>
|
||
public static ulong ComputeFingerprint(int mapId)
|
||
{
|
||
var hasher = HashUtility.CreateXxHash3();
|
||
|
||
// TileData flag tables — same projection trick as before: just the Flags ulong
|
||
// from each entry, written little-endian into a contiguous byte buffer. The
|
||
// struct itself has a string Name (reference) whose object identity isn't
|
||
// stable across runs, so MemoryMarshal.Cast over the whole struct would drift.
|
||
var landTable = TileData.LandTable;
|
||
var itemTable = TileData.ItemTable;
|
||
var bytes = new byte[(landTable.Length + itemTable.Length) * sizeof(ulong)];
|
||
var span = bytes.AsSpan();
|
||
|
||
for (var i = 0; i < landTable.Length; i++)
|
||
{
|
||
BinaryPrimitives.WriteUInt64LittleEndian(span[(i * 8)..], (ulong)landTable[i].Flags);
|
||
}
|
||
var itemOffset = landTable.Length * 8;
|
||
for (var i = 0; i < itemTable.Length; i++)
|
||
{
|
||
BinaryPrimitives.WriteUInt64LittleEndian(span[(itemOffset + i * 8)..], (ulong)itemTable[i].Flags);
|
||
}
|
||
hasher.Append(bytes);
|
||
|
||
// Map files (mapX.mul / .uop, staidxX.mul, staticsX.mul). TileMatrix already
|
||
// streamed them through XxHash3 once at construction; mix the result in.
|
||
var map = Map.Maps[mapId];
|
||
if (map != null && map != Map.Internal && map.Tiles != null)
|
||
{
|
||
Span<byte> mapHashBytes = stackalloc byte[sizeof(ulong)];
|
||
BinaryPrimitives.WriteUInt64LittleEndian(mapHashBytes, map.Tiles.MapFilesFingerprint);
|
||
hasher.Append(mapHashBytes);
|
||
}
|
||
|
||
return hasher.GetCurrentHashAsUInt64();
|
||
}
|
||
|
||
/// <summary>
|
||
/// Writes the file: header (with placeholder IndexOffset) → chunks (offsets recorded)
|
||
/// → index trailer → patches the header IndexOffset. <paramref name="chunkCount"/> must
|
||
/// equal the actual number of chunks <paramref name="next"/> will yield.
|
||
/// </summary>
|
||
public static void Write(string path, uint mapId, uint chunkCount, ChunkEnumerator next)
|
||
{
|
||
Directory.CreateDirectory(Path.GetDirectoryName(path) ?? ".");
|
||
|
||
// Initial estimate: base record + a modest strata budget per chunk. Coastline
|
||
// chunks add another ~2.5 KB (swim layer) but they're a small fraction of any
|
||
// map; the writer grows on overflow so under-estimating just causes a few
|
||
// realloc/copy cycles during the bake — not a correctness issue.
|
||
var capacity = HeaderSize
|
||
+ (BytesPerChunkBase + 256) * (int)chunkCount
|
||
+ IndexEntryBytes * (int)chunkCount;
|
||
var buffer = new byte[capacity];
|
||
var w = new BufferWriter(buffer, prefixStr: false);
|
||
|
||
w.Write(Magic);
|
||
w.Write(FormatVersion);
|
||
w.Write(mapId);
|
||
w.Write(ComputeFingerprint((int)mapId));
|
||
w.Write((ulong)DateTime.UtcNow.Ticks);
|
||
w.Write(chunkCount);
|
||
w.Write(0UL); // IndexOffset placeholder, patched after chunks
|
||
|
||
var indexEntries = new (ulong key, ulong offset, uint length)[chunkCount];
|
||
var written = 0u;
|
||
while (next(out var chunkX, out var chunkY, out var chunk))
|
||
{
|
||
if (written >= chunkCount)
|
||
{
|
||
throw new InvalidOperationException(
|
||
$"StepCacheFile.Write: enumerator yielded more than the declared {chunkCount} chunks"
|
||
);
|
||
}
|
||
var chunkOffset = (ulong)w.Position;
|
||
WriteChunk(w, chunkX, chunkY, chunk);
|
||
var chunkLength = (uint)((ulong)w.Position - chunkOffset);
|
||
indexEntries[written] = (PackChunkKey(chunkX, chunkY), chunkOffset, chunkLength);
|
||
written++;
|
||
}
|
||
|
||
if (written != chunkCount)
|
||
{
|
||
throw new InvalidOperationException(
|
||
$"StepCacheFile.Write: declared {chunkCount} chunks but enumerator yielded {written}"
|
||
);
|
||
}
|
||
|
||
var indexOffset = (ulong)w.Position;
|
||
for (var i = 0u; i < chunkCount; i++)
|
||
{
|
||
w.Write(indexEntries[i].key);
|
||
w.Write(indexEntries[i].offset);
|
||
w.Write(indexEntries[i].length);
|
||
}
|
||
|
||
// Patch IndexOffset on the writer's current backing buffer (BufferWriter may
|
||
// have grown during chunk writes; the original `buffer` ref is stale after grow).
|
||
var liveBuffer = w.Buffer;
|
||
BinaryPrimitives.WriteUInt64LittleEndian(liveBuffer.AsSpan(IndexOffsetFieldPosition, 8), indexOffset);
|
||
|
||
var totalBytes = (int)w.Position;
|
||
File.WriteAllBytes(path, liveBuffer.AsSpan(0, totalBytes).ToArray());
|
||
}
|
||
|
||
/// <summary>
|
||
/// Opens a .swb file and reads only its header + chunk-offset index. Returns null on
|
||
/// missing file, magic / version mismatch, or Fingerprint mismatch (a stale bake
|
||
/// against a freshly patched client). Callers own disposal of the returned reader.
|
||
/// </summary>
|
||
public static LazyReader OpenForLazy(string path)
|
||
{
|
||
if (!File.Exists(path))
|
||
{
|
||
return null;
|
||
}
|
||
|
||
FileStream stream = null;
|
||
try
|
||
{
|
||
stream = new FileStream(
|
||
path,
|
||
FileMode.Open,
|
||
FileAccess.Read,
|
||
FileShare.Read | FileShare.Delete
|
||
);
|
||
|
||
Span<byte> headerBuf = stackalloc byte[HeaderSize];
|
||
if (stream.Read(headerBuf) != HeaderSize)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
var magic = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf);
|
||
if (magic != Magic)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
var version = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[4..]);
|
||
if (version < MinSupportedVersion || version > FormatVersion)
|
||
{
|
||
// Below the minimum supported version: treat as missing. Older files
|
||
// get silently overwritten on the next SaveToFile / BakeMap.
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
var mapId = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[8..]);
|
||
var fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[12..]);
|
||
var bakeTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[20..]);
|
||
var chunkCount = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[28..]);
|
||
var indexOffset = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[32..]);
|
||
|
||
if (fingerprint != ComputeFingerprint((int)mapId))
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
// Read the chunk-offset index in one shot.
|
||
var indexBytes = (int)chunkCount * IndexEntryBytes;
|
||
var indexBuf = new byte[indexBytes];
|
||
stream.Position = (long)indexOffset;
|
||
if (stream.Read(indexBuf, 0, indexBytes) != indexBytes)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
var offsets = new Dictionary<ulong, (ulong offset, uint length)>((int)chunkCount);
|
||
for (var i = 0; i < chunkCount; i++)
|
||
{
|
||
var entry = indexBuf.AsSpan(i * IndexEntryBytes);
|
||
var key = BinaryPrimitives.ReadUInt64LittleEndian(entry);
|
||
var off = BinaryPrimitives.ReadUInt64LittleEndian(entry[8..]);
|
||
var len = BinaryPrimitives.ReadUInt32LittleEndian(entry[16..]);
|
||
offsets[key] = (off, len);
|
||
}
|
||
|
||
return new LazyReader(stream, mapId, fingerprint, bakeTimestamp, chunkCount, offsets);
|
||
}
|
||
catch
|
||
{
|
||
stream?.Dispose();
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private static ulong PackChunkKey(int chunkX, int chunkY) => ((ulong)(uint)chunkX << 32) | (uint)chunkY;
|
||
|
||
/// <summary>
|
||
/// Predicted directional-Z for one cell/direction: the cell's own SourceZ when the
|
||
/// direction is walkable/wet (mask bit set), else 0 — matching the baker, which leaves
|
||
/// non-walkable directional slots at their zero-initialized default
|
||
/// (StepProbe.ComputeMaskAt clears walkZs/swimZs and writes only on a successful step).
|
||
/// </summary>
|
||
internal static sbyte Predict(byte dirMaskByte, int bit, sbyte sourceZ) =>
|
||
(dirMaskByte >> bit & 1) != 0 ? sourceZ : (sbyte)0;
|
||
|
||
/// <summary>
|
||
/// Residual of an absolute directional-Z against its prediction. Unchecked two's-complement
|
||
/// so the transform is byte-exact for ALL sbyte inputs (no value-range constraint).
|
||
/// </summary>
|
||
internal static sbyte EncodeResidual(sbyte z, sbyte predict) => unchecked((sbyte)(z - predict));
|
||
|
||
/// <summary>Inverse of <see cref="EncodeResidual"/>: absolute directional-Z = predict + residual.</summary>
|
||
internal static sbyte DecodeZ(sbyte predict, sbyte residual) => unchecked((sbyte)(predict + residual));
|
||
|
||
/// <summary>
|
||
/// The 16 base directional-Z arrays in canonical order: walk N..NW (indices 0-7),
|
||
/// then swim N..NW (8-15). Index d uses WalkMask (d < 8) or WetMask (d >= 8)
|
||
/// with direction bit (d & 7). Allocates a 16-slot reference array (bake/read time only).
|
||
/// </summary>
|
||
private static sbyte[][] GetBaseZArrays(StepChunk c) => new[]
|
||
{
|
||
c.WalkZN, c.WalkZNE, c.WalkZE, c.WalkZSE, c.WalkZS, c.WalkZSW, c.WalkZW, c.WalkZNW,
|
||
c.SwimZN, c.SwimZNE, c.SwimZE, c.SwimZSE, c.SwimZS, c.SwimZSW, c.SwimZW, c.SwimZNW,
|
||
};
|
||
|
||
private static void WriteChunk(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
|
||
{
|
||
w.Write((ushort)chunkX);
|
||
w.Write((ushort)chunkY);
|
||
w.Write((uint)chunk.BuiltMultisVersion);
|
||
|
||
// Kind: 0 = Full, 2 = Uniform. A uniform chunk (no strata, no swim layer, all 19 base
|
||
// arrays constant) stores one cell's worth of data (~28-byte record total).
|
||
if (chunk.IsUniform())
|
||
{
|
||
w.Write(KindUniform);
|
||
w.Write(chunk.WalkMask[0]);
|
||
w.Write(chunk.WetMask[0]);
|
||
w.Write((byte)chunk.SourceZ[0]);
|
||
w.Write((byte)chunk.WalkZN[0]);
|
||
w.Write((byte)chunk.WalkZNE[0]);
|
||
w.Write((byte)chunk.WalkZE[0]);
|
||
w.Write((byte)chunk.WalkZSE[0]);
|
||
w.Write((byte)chunk.WalkZS[0]);
|
||
w.Write((byte)chunk.WalkZSW[0]);
|
||
w.Write((byte)chunk.WalkZW[0]);
|
||
w.Write((byte)chunk.WalkZNW[0]);
|
||
w.Write((byte)chunk.SwimZN[0]);
|
||
w.Write((byte)chunk.SwimZNE[0]);
|
||
w.Write((byte)chunk.SwimZE[0]);
|
||
w.Write((byte)chunk.SwimZSE[0]);
|
||
w.Write((byte)chunk.SwimZS[0]);
|
||
w.Write((byte)chunk.SwimZSW[0]);
|
||
w.Write((byte)chunk.SwimZW[0]);
|
||
w.Write((byte)chunk.SwimZNW[0]);
|
||
return;
|
||
}
|
||
|
||
w.Write(KindFull); // Full
|
||
|
||
var strataOffsetByCell = chunk.GetStrataOffsetByCellForSerialization();
|
||
var strataData = chunk.GetStrataDataForSerialization();
|
||
var hasStrata = strataOffsetByCell != null;
|
||
var hasSwimLayer = chunk.HasSwimLayer;
|
||
w.Write((byte)(hasStrata ? 1 : 0));
|
||
w.Write((byte)(hasSwimLayer ? 1 : 0));
|
||
|
||
// Predictive-Z: each base directional Z array is stored as a masked residual against
|
||
// SourceZ. Bit d of ZArrayMask is set only when array d differs from its prediction
|
||
// somewhere; cleared arrays are omitted and rebuilt from mask+SourceZ at read.
|
||
var zArrays = GetBaseZArrays(chunk);
|
||
ushort zArrayMask = 0;
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
var z = zArrays[d];
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
if (z[cell] != Predict(dirMask[cell], bit, chunk.SourceZ[cell]))
|
||
{
|
||
zArrayMask |= (ushort)(1 << d);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
w.Write(zArrayMask);
|
||
|
||
w.Write(chunk.WalkMask);
|
||
w.Write(chunk.WetMask);
|
||
WriteSBytes(w, chunk.SourceZ);
|
||
|
||
Span<sbyte> residual = stackalloc sbyte[StepChunk.CellsPerChunk];
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
if ((zArrayMask >> d & 1) == 0)
|
||
{
|
||
continue;
|
||
}
|
||
var z = zArrays[d];
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
residual[cell] = EncodeResidual(z[cell], Predict(dirMask[cell], bit, chunk.SourceZ[cell]));
|
||
}
|
||
w.Write(MemoryMarshal.Cast<sbyte, byte>(residual));
|
||
}
|
||
|
||
if (hasSwimLayer)
|
||
{
|
||
WriteSBytes(w, chunk.SwimSourceZ);
|
||
w.Write(chunk.SwimMask);
|
||
WriteSBytes(w, chunk.SwimZN_Layer);
|
||
WriteSBytes(w, chunk.SwimZNE_Layer);
|
||
WriteSBytes(w, chunk.SwimZE_Layer);
|
||
WriteSBytes(w, chunk.SwimZSE_Layer);
|
||
WriteSBytes(w, chunk.SwimZS_Layer);
|
||
WriteSBytes(w, chunk.SwimZSW_Layer);
|
||
WriteSBytes(w, chunk.SwimZW_Layer);
|
||
WriteSBytes(w, chunk.SwimZNW_Layer);
|
||
}
|
||
|
||
if (hasStrata)
|
||
{
|
||
// 256 × u16 offsets, then u32 length-prefixed strata byte array.
|
||
for (var i = 0; i < StepChunk.CellsPerChunk; i++)
|
||
{
|
||
w.Write(strataOffsetByCell[i]);
|
||
}
|
||
var dataLen = (uint)(strataData?.Length ?? 0);
|
||
w.Write(dataLen);
|
||
if (dataLen > 0)
|
||
{
|
||
w.Write(strataData);
|
||
}
|
||
}
|
||
}
|
||
|
||
private static StepChunk ReadChunk(byte[] buffer)
|
||
{
|
||
var r = new BufferReader(buffer);
|
||
// Skip ChunkX + ChunkY (already known via the index lookup).
|
||
r.ReadUShort();
|
||
r.ReadUShort();
|
||
var multisVersion = (int)r.ReadUInt();
|
||
var kind = r.ReadByte();
|
||
|
||
var chunk = new StepChunk { BuiltMultisVersion = multisVersion };
|
||
|
||
if (kind == KindUniform) // Uniform — one cell's worth of the 19 base arrays, fill all 256 cells.
|
||
{
|
||
Array.Fill(chunk.WalkMask, r.ReadByte());
|
||
Array.Fill(chunk.WetMask, r.ReadByte());
|
||
Array.Fill(chunk.SourceZ, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZN, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZNE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZSE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZS, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZSW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZNW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZN, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZNE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZSE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZS, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZSW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZNW, (sbyte)r.ReadByte());
|
||
return chunk;
|
||
}
|
||
|
||
var hasStrata = r.ReadByte() != 0;
|
||
var hasSwimLayer = r.ReadByte() != 0;
|
||
var zArrayMask = r.ReadUShort();
|
||
|
||
r.Read(chunk.WalkMask);
|
||
r.Read(chunk.WetMask);
|
||
ReadSBytes(r, chunk.SourceZ);
|
||
|
||
// Predictive-Z reconstruction: present arrays carry residuals (z = predict + residual);
|
||
// absent arrays are synthesized from mask+SourceZ (z = predict, residual implicitly 0).
|
||
var zArrays = GetBaseZArrays(chunk);
|
||
Span<sbyte> residual = stackalloc sbyte[StepChunk.CellsPerChunk];
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
var z = zArrays[d];
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
if ((zArrayMask >> d & 1) != 0)
|
||
{
|
||
r.Read(MemoryMarshal.Cast<sbyte, byte>(residual));
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
z[cell] = DecodeZ(Predict(dirMask[cell], bit, chunk.SourceZ[cell]), residual[cell]);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
z[cell] = Predict(dirMask[cell], bit, chunk.SourceZ[cell]);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (hasSwimLayer)
|
||
{
|
||
chunk.AllocateSwimLayer();
|
||
ReadSBytes(r, chunk.SwimSourceZ);
|
||
r.Read(chunk.SwimMask);
|
||
ReadSBytes(r, chunk.SwimZN_Layer);
|
||
ReadSBytes(r, chunk.SwimZNE_Layer);
|
||
ReadSBytes(r, chunk.SwimZE_Layer);
|
||
ReadSBytes(r, chunk.SwimZSE_Layer);
|
||
ReadSBytes(r, chunk.SwimZS_Layer);
|
||
ReadSBytes(r, chunk.SwimZSW_Layer);
|
||
ReadSBytes(r, chunk.SwimZW_Layer);
|
||
ReadSBytes(r, chunk.SwimZNW_Layer);
|
||
}
|
||
|
||
if (hasStrata)
|
||
{
|
||
var offsets = new ushort[StepChunk.CellsPerChunk];
|
||
for (var i = 0; i < offsets.Length; i++)
|
||
{
|
||
offsets[i] = r.ReadUShort();
|
||
}
|
||
var dataLen = (int)r.ReadUInt();
|
||
var data = new byte[dataLen];
|
||
if (dataLen > 0)
|
||
{
|
||
r.Read(data);
|
||
}
|
||
chunk.SetStrata(offsets, data);
|
||
}
|
||
|
||
return chunk;
|
||
}
|
||
|
||
private static void WriteSBytes(BufferWriter w, sbyte[] arr) =>
|
||
w.Write(MemoryMarshal.Cast<sbyte, byte>(arr.AsSpan()));
|
||
|
||
private static void ReadSBytes(BufferReader r, sbyte[] arr) =>
|
||
r.Read(MemoryMarshal.Cast<sbyte, byte>(arr.AsSpan()));
|
||
|
||
/// <summary>
|
||
/// Open handle on a .swb file. Holds the FileStream + chunk-offset index. Chunks are
|
||
/// fetched on demand via <see cref="TryReadChunk"/>; only the records actually queried
|
||
/// are ever materialized. Dispose releases the underlying stream.
|
||
/// </summary>
|
||
internal sealed class LazyReader : IDisposable
|
||
{
|
||
private FileStream _stream;
|
||
private readonly Dictionary<ulong, (ulong offset, uint length)> _offsets;
|
||
private byte[] _buffer;
|
||
|
||
public uint MapId { get; }
|
||
public ulong Fingerprint { get; }
|
||
public ulong BakeTimestamp { get; }
|
||
public uint ChunkCount { get; }
|
||
public int IndexedChunkCount => _offsets.Count;
|
||
|
||
public bool Has(int chunkX, int chunkY) => _offsets.ContainsKey(PackChunkKey(chunkX, chunkY));
|
||
|
||
/// <summary>
|
||
/// Enumerates every (chunkX, chunkY) coordinate the file holds. Used by
|
||
/// <see cref="StepCache"/> when preload is enabled to materialize all chunks
|
||
/// upfront instead of on first query.
|
||
/// </summary>
|
||
public IEnumerable<(int chunkX, int chunkY)> EnumerateChunkCoords()
|
||
{
|
||
foreach (var key in _offsets.Keys)
|
||
{
|
||
yield return ((int)(key >> 32), (int)(key & 0xFFFFFFFF));
|
||
}
|
||
}
|
||
|
||
internal LazyReader(
|
||
FileStream stream, uint mapId, ulong fingerprint, ulong bakeTimestamp,
|
||
uint chunkCount, Dictionary<ulong, (ulong offset, uint length)> offsets
|
||
)
|
||
{
|
||
_stream = stream;
|
||
MapId = mapId;
|
||
Fingerprint = fingerprint;
|
||
BakeTimestamp = bakeTimestamp;
|
||
ChunkCount = chunkCount;
|
||
_offsets = offsets;
|
||
_buffer = new byte[BytesPerChunkBase];
|
||
}
|
||
|
||
/// <summary>
|
||
/// Returns the chunk record at (<paramref name="chunkX"/>, <paramref name="chunkY"/>)
|
||
/// from the file, or null if the file doesn't contain it. Single seek + bulk read,
|
||
/// sized exactly to the chunk's recorded length (which varies with strata size).
|
||
/// </summary>
|
||
public StepChunk TryReadChunk(int chunkX, int chunkY)
|
||
{
|
||
if (_stream == null)
|
||
{
|
||
return null;
|
||
}
|
||
|
||
var key = PackChunkKey(chunkX, chunkY);
|
||
if (!_offsets.TryGetValue(key, out var entry))
|
||
{
|
||
return null;
|
||
}
|
||
|
||
// Grow the scratch buffer if this chunk's record is larger than what we have.
|
||
// Common case: chunks fit in BytesPerChunkBase; only multi-Z-heavy chunks grow.
|
||
if (entry.length > _buffer.Length)
|
||
{
|
||
_buffer = new byte[entry.length];
|
||
}
|
||
|
||
_stream.Position = (long)entry.offset;
|
||
var read = _stream.Read(_buffer, 0, (int)entry.length);
|
||
return read < (int)entry.length ? null : ReadChunk(_buffer);
|
||
}
|
||
|
||
public void Dispose()
|
||
{
|
||
_stream?.Dispose();
|
||
_stream = null;
|
||
_buffer = null;
|
||
}
|
||
}
|
||
}
|