79df316163
SaveToFile walked _keysList twice - once to count the map's chunks, then again through a ChunkEnumerator closure to emit them - because Write needed the count up front to size its index array. It now collects the map's chunks into a PooledRefList in one pass and hands Write a span, so the count is just span.Length. That deletes the ChunkEnumerator delegate, the closure over the list enumerator, and both InvalidOperationException throws, which existed only to police the delegate's "yield exactly chunkCount chunks" contract - a contract a span makes unrepresentable. Write now patches the header's IndexOffset by seeking back to it rather than reaching into the writer's live buffer with BinaryPrimitives. This also removes the IndexOffsetFieldPosition constant, which hard-coded the field's byte offset and had to be kept in sync with the header layout by hand. Tests: StepCacheFileV6/V7/V8Tests were named for the format version that introduced each transform, and the format is now v9 - so all three names described formats the loader rejects outright. Each file also carried its own copy of the chunk builders, the round-trip plumbing, and the assertion helper, plus a near-identical "an older version is rejected" test whose comment cited a MinSupportedVersion that had since moved. They are now one StepCacheFileFormatTests named for the behavior it covers - predictive-Z elision, compression, the compact index - with shared builders, a single AssertIdentical that checks the swim and strata trailers too (the old AssertBaseEqual silently skipped both), and the three rejection tests folded into one theory that also covers a future version. Added a zero-chunk case, which the old delegate-based writer had no coverage for. Verified non-vacuous: injecting an off-by-one into the IndexOffset patch fails 15 of the 123 tests. Also reattaches a doc comment in StepCacheFileTests that had drifted off LazyReaderHit_BypassesMissTrackerOnFirstTouch and stacked onto the test above it, and drops its reference to the long-gone "v3 file format". Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
776 lines
32 KiB
C#
776 lines
32 KiB
C#
using System;
|
||
using System.Buffers.Binary;
|
||
using System.Collections.Generic;
|
||
using System.IO;
|
||
using System.IO.Compression;
|
||
using System.Runtime.InteropServices;
|
||
using Server.Compression;
|
||
|
||
namespace Server.Engines.Pathing.Cache;
|
||
|
||
/// <summary>
|
||
/// Binary serializer and reader for the step cache, so a shard can warm-start instead of building
|
||
/// chunks on the first pathfind through each region. Opening a file reads only the header and chunk
|
||
/// index; a chunk record is seeked and inflated when the cache actually asks for it, which keeps
|
||
/// resident memory bounded by MaxResidentChunks no matter how large the file is.
|
||
///
|
||
/// File layout (little-endian, BufferWriter / BufferReader convention):
|
||
///
|
||
/// Header (40 bytes):
|
||
/// u32 Magic = 0x42575300 ('SWB\0')
|
||
/// u32 Version = FormatVersion
|
||
/// u32 MapId
|
||
/// u64 Fingerprint XxHash3 over tiledata.mul and the map's own .mul / .uop files.
|
||
/// Detects both a client patch that shifts tile flags and a map edit
|
||
/// that rewrites the terrain; see ComputeFingerprint. The .mul format
|
||
/// carries no CRC of its own, so hashing is the only way to catch either.
|
||
/// u64 BakeTimestamp DateTime.UtcNow.Ticks at write time. Informational.
|
||
/// u32 ChunkCount
|
||
/// u64 IndexOffset Where the index trailer begins.
|
||
///
|
||
/// Per chunk (ChunkCount times, variable size):
|
||
/// u32 UncompressedLen Size of the inflated record body below.
|
||
/// byte[] Payload The record body, libdeflate-compressed — or stored raw when
|
||
/// compression didn't shrink it, as happens with tiny Uniform
|
||
/// records. The reader tells the two apart by comparing the payload
|
||
/// length against UncompressedLen.
|
||
///
|
||
/// Record body (after inflate):
|
||
/// u16 ChunkX
|
||
/// u16 ChunkY
|
||
/// u32 BuiltMultisVersion Reserved, always 0 — chunks are static-only.
|
||
/// u8 Kind 0 = Full; 2 = Uniform
|
||
/// // Uniform (Kind == 2): ~28-byte record — all 256 cells share these single values:
|
||
/// byte walkMask, wetMask; sbyte sourceZ; sbyte walkZ_N..NW (8); sbyte swimZ_N..NW (8)
|
||
/// // Full (Kind == 0) body:
|
||
/// u8 HasStrata 0 = single-Z chunk (no strata trailer); 1 = strata trailer follows
|
||
/// u8 HasSwimLayer 0 = no shore cells (no swim trailer); 1 = swim trailer follows
|
||
/// u16 ZArrayMask bit d set => base directional Z array d is present below as a
|
||
/// residual[256] block; cleared => array equals its prediction and is
|
||
/// omitted (synthesized at read). bits 0-7 = WalkZ N..NW (predicted via
|
||
/// WalkMask), bits 8-15 = SwimZ N..NW (predicted via WetMask).
|
||
/// byte WalkMask[256]
|
||
/// byte WetMask[256]
|
||
/// sbyte SourceZ[256]
|
||
/// // For each d in 0..15 with ZArrayMask bit d set, in N,NE,E,SE,S,SW,W,NW order
|
||
/// // (walk arrays first, then swim):
|
||
/// sbyte residual_d[256] reconstruct: Z_d[c] = (mask bit set ? SourceZ[c] : 0) + residual_d[c]
|
||
/// // Swim layer trailer — only when HasSwimLayer == 1 (chunks containing shore cells):
|
||
/// sbyte SwimSourceZ[256] (NoSwimLayerCell sentinel = sbyte.MinValue)
|
||
/// byte SwimMask[256] (per-cell swim mask baked at SwimSourceZ)
|
||
/// sbyte SwimZN_Layer[256]..SwimZNW_Layer[256] (8 arrays, dest-Z at swim perspective)
|
||
/// // Strata trailer — only when HasStrata == 1:
|
||
/// u16 StrataOffsetByCell[256] (NoStrata sentinel = 0xFFFF)
|
||
/// u32 StrataDataLength
|
||
/// byte StrataData[StrataDataLength]
|
||
/// For each multi-Z cell: u8 stratumCount, then stratumCount × Stratum (19 bytes):
|
||
/// sbyte zCenter
|
||
/// byte walkMask, wetMask
|
||
/// sbyte walkZ_N..NW (8)
|
||
/// sbyte swimZ_N..NW (8)
|
||
///
|
||
/// Index trailer (8 × ChunkCount bytes), in record write order:
|
||
/// For each chunk: { u32 packedKey = (ChunkX << 16) | ChunkY, u32 recordLength }
|
||
/// The file offset is not stored — reconstructed as a cumulative sum of recordLength
|
||
/// starting at HeaderSize (the first record sits immediately after the header).
|
||
///
|
||
/// A chunk's fixed portion runs ~783 bytes, and each directional-Z array that survives prediction
|
||
/// adds 256 more, so a Full record lands between ~783 bytes and ~4 KB. The strata trailer adds
|
||
/// 516 bytes plus roughly 30 per multi-Z cell. LastTouchedTicks is deliberately not persisted —
|
||
/// LRU state means nothing across a restart.
|
||
///
|
||
/// Files below <see cref="MinSupportedVersion"/> are treated as missing and overwritten on the
|
||
/// next save. The cache regenerates from the map data, so a format bump only costs a one-time
|
||
/// re-bake.
|
||
/// </summary>
|
||
internal static class StepCacheFile
|
||
{
|
||
public const uint Magic = 0x42575300; // 'SWB\0'
|
||
|
||
public const uint FormatVersion = 9;
|
||
|
||
/// <summary>
|
||
/// Oldest format this binary will load. Anything older is treated as missing rather than
|
||
/// migrated: the cache is fully regenerable from the map data, so a re-bake is always
|
||
/// available and always correct.
|
||
/// </summary>
|
||
public const uint MinSupportedVersion = 9;
|
||
|
||
// Record discriminator. 1 is reserved.
|
||
private const byte KindFull = 0;
|
||
private const byte KindUniform = 2;
|
||
|
||
private const int HeaderSize =
|
||
sizeof(uint) // Magic
|
||
+ sizeof(uint) // Version
|
||
+ sizeof(uint) // MapId
|
||
+ sizeof(ulong) // Fingerprint
|
||
+ sizeof(ulong) // BakeTimestamp
|
||
+ sizeof(uint) // ChunkCount
|
||
+ sizeof(ulong); // IndexOffset
|
||
|
||
// One index entry: u32 packedKey ((chunkX << 16) | chunkY) + u32 recordLength. The file offset
|
||
// isn't stored — entries sit in record write order, so the reader rebuilds each offset as a
|
||
// running sum of the lengths before it, starting at HeaderSize.
|
||
private const int IndexEntryBytes = sizeof(uint) + sizeof(uint);
|
||
|
||
/// <summary>A chunk record minus its optional strata and swim trailers.</summary>
|
||
private const int BytesPerChunkBase =
|
||
sizeof(ushort) + sizeof(ushort) + sizeof(uint)
|
||
+ sizeof(byte) + sizeof(byte) + sizeof(byte) // Kind + HasStrata + HasSwimLayer
|
||
+ sizeof(ushort) // ZArrayMask
|
||
+ StepChunk.CellsPerChunk // WalkMask
|
||
+ StepChunk.CellsPerChunk // WetMask
|
||
+ StepChunk.CellsPerChunk // SourceZ
|
||
+ 8 * StepChunk.CellsPerChunk // WalkZ[8]
|
||
+ 8 * StepChunk.CellsPerChunk; // SwimZ[8]
|
||
|
||
/// <summary>
|
||
/// Reads just a .swb file's fingerprint — 20 bytes, no chunk data. False if the file is
|
||
/// missing, isn't a .swb, or is a version this binary can't load.
|
||
/// </summary>
|
||
public static bool TryReadFingerprint(string path, out ulong fingerprint)
|
||
{
|
||
fingerprint = 0;
|
||
if (!File.Exists(path))
|
||
{
|
||
return false;
|
||
}
|
||
|
||
try
|
||
{
|
||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
|
||
Span<byte> buf = stackalloc byte[20];
|
||
if (stream.Read(buf) < 20)
|
||
{
|
||
return false;
|
||
}
|
||
if (BinaryPrimitives.ReadUInt32LittleEndian(buf) != Magic)
|
||
{
|
||
return false;
|
||
}
|
||
var version = BinaryPrimitives.ReadUInt32LittleEndian(buf[4..]);
|
||
if (version < MinSupportedVersion || version > FormatVersion)
|
||
{
|
||
return false;
|
||
}
|
||
// mapId is at buf[8..12], we skip; hash is at buf[12..20].
|
||
fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(buf[12..]);
|
||
return true;
|
||
}
|
||
catch
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Hashes the inputs a bake depends on: <c>tiledata.mul</c> and the map's own .mul / .uop
|
||
/// files. A file carrying a stale hash is refused at open time, which is what catches a client
|
||
/// patch that shifts tile flags or a map editor that rewrites the terrain. Neither format has a
|
||
/// CRC of its own, so hashing is the only signal available.
|
||
///
|
||
/// This must hash the FILES, never the in-memory <see cref="TileData.LandTable"/> /
|
||
/// <see cref="TileData.ItemTable"/>. The server patches those tables at runtime (ItemFixes,
|
||
/// LOSBlocker, PotionKeg, CTF), so a hash of the live tables changes depending on when it is
|
||
/// taken — useless as a fingerprint. Those server-side patches apply identically every boot and
|
||
/// deliberately do NOT invalidate the cache; if you change one, run [PathCacheClear or bump
|
||
/// <see cref="FormatVersion"/> yourself.
|
||
/// </summary>
|
||
public static ulong ComputeFingerprint(int mapId)
|
||
{
|
||
var hasher = HashUtility.CreateXxHash3();
|
||
|
||
Span<byte> tileDataBytes = stackalloc byte[sizeof(ulong)];
|
||
BinaryPrimitives.WriteUInt64LittleEndian(tileDataBytes, TileDataFileFingerprint());
|
||
hasher.Append(tileDataBytes);
|
||
|
||
// TileMatrix already streamed the map files through XxHash3 when it was built; reuse that
|
||
// rather than re-reading them.
|
||
var map = Map.Maps[mapId];
|
||
if (map != null && map != Map.Internal && map.Tiles != null)
|
||
{
|
||
Span<byte> mapHashBytes = stackalloc byte[sizeof(ulong)];
|
||
BinaryPrimitives.WriteUInt64LittleEndian(mapHashBytes, map.Tiles.MapFilesFingerprint);
|
||
hasher.Append(mapHashBytes);
|
||
}
|
||
|
||
return hasher.GetCurrentHashAsUInt64();
|
||
}
|
||
|
||
private static ulong _tileDataFileFingerprint;
|
||
private static bool _tileDataFileFingerprintComputed;
|
||
|
||
/// <summary>
|
||
/// XxHash3 of the raw <c>tiledata.mul</c> bytes, computed once — the file can't change while
|
||
/// the server runs. Returns 0 when the file is absent, which only happens in stripped test
|
||
/// hosts; a real server can't boot without it, and 0 is a fine deterministic stand-in.
|
||
/// </summary>
|
||
private static ulong TileDataFileFingerprint()
|
||
{
|
||
if (_tileDataFileFingerprintComputed)
|
||
{
|
||
return _tileDataFileFingerprint;
|
||
}
|
||
|
||
var path = Core.FindDataFile("tiledata.mul", false);
|
||
if (path != null)
|
||
{
|
||
using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||
var hasher = HashUtility.CreateXxHash3();
|
||
hasher.Append(fs);
|
||
_tileDataFileFingerprint = hasher.GetCurrentHashAsUInt64();
|
||
}
|
||
|
||
_tileDataFileFingerprintComputed = true;
|
||
return _tileDataFileFingerprint;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Writes the map's chunks to <paramref name="path"/>: header, then one record per chunk, then
|
||
/// the index trailer. IndexOffset isn't known until the records are down, so it goes in as a
|
||
/// placeholder and gets patched by seeking back to it.
|
||
/// </summary>
|
||
public static void Write(string path, uint mapId, ReadOnlySpan<(int chunkX, int chunkY, StepChunk chunk)> chunks)
|
||
{
|
||
Directory.CreateDirectory(Path.GetDirectoryName(path) ?? ".");
|
||
|
||
// A rough estimate: the base record plus a small strata budget per chunk. Coastline chunks
|
||
// run ~2.5 KB over it for their swim layer, but they're a small share of any map, and the
|
||
// writer grows on overflow — under-estimating costs a few reallocs during a bake, nothing more.
|
||
var capacity = HeaderSize + (BytesPerChunkBase + 256 + IndexEntryBytes) * chunks.Length;
|
||
var w = new BufferWriter(new byte[capacity], prefixStr: false);
|
||
|
||
w.Write(Magic);
|
||
w.Write(FormatVersion);
|
||
w.Write(mapId);
|
||
w.Write(ComputeFingerprint((int)mapId));
|
||
w.Write((ulong)DateTime.UtcNow.Ticks);
|
||
w.Write((uint)chunks.Length);
|
||
var indexOffsetPosition = w.Position;
|
||
w.Write(0UL); // patched below, once the records are written and the index position is known
|
||
|
||
// Each record is built into recordScratch, compressed into compScratch, then framed as
|
||
// [u32 uncompressedLen][payload].
|
||
var packer = Deflate.Maximum;
|
||
var recordScratch = new byte[BytesPerChunkBase + 1024];
|
||
var compScratch = new byte[packer.MaxPackSize(recordScratch.Length)];
|
||
|
||
// Record lengths only — the index stores no offsets, so the reader rebuilds them by
|
||
// summing these in order.
|
||
var lengths = new uint[chunks.Length];
|
||
for (var i = 0; i < chunks.Length; i++)
|
||
{
|
||
var (chunkX, chunkY, chunk) = chunks[i];
|
||
var start = w.Position;
|
||
WriteChunk(w, chunkX, chunkY, chunk, packer, ref recordScratch, ref compScratch);
|
||
lengths[i] = (uint)(w.Position - start);
|
||
}
|
||
|
||
var indexOffset = (ulong)w.Position;
|
||
for (var i = 0; i < chunks.Length; i++)
|
||
{
|
||
var (chunkX, chunkY, _) = chunks[i];
|
||
w.Write((uint)((chunkX & 0xFFFF) << 16 | chunkY & 0xFFFF));
|
||
w.Write(lengths[i]);
|
||
}
|
||
|
||
var totalBytes = (int)w.Position;
|
||
|
||
w.Seek(indexOffsetPosition, SeekOrigin.Begin);
|
||
w.Write(indexOffset);
|
||
|
||
// w.Buffer, not the array handed to the constructor: BufferWriter reallocates on growth,
|
||
// which leaves that original reference pointing at a stale array.
|
||
File.WriteAllBytes(path, w.Buffer.AsSpan(0, totalBytes).ToArray());
|
||
}
|
||
|
||
/// <summary>
|
||
/// Opens a .swb file, reading only its header and chunk index. Null if the file is missing,
|
||
/// isn't a loadable .swb, or is a stale bake whose fingerprint no longer matches the live tile
|
||
/// and map data. The caller owns the returned reader.
|
||
/// </summary>
|
||
public static LazyReader OpenForLazy(string path)
|
||
{
|
||
if (!File.Exists(path))
|
||
{
|
||
return null;
|
||
}
|
||
|
||
FileStream stream = null;
|
||
try
|
||
{
|
||
stream = new FileStream(
|
||
path,
|
||
FileMode.Open,
|
||
FileAccess.Read,
|
||
FileShare.Read | FileShare.Delete
|
||
);
|
||
|
||
Span<byte> headerBuf = stackalloc byte[HeaderSize];
|
||
if (stream.Read(headerBuf) != HeaderSize)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
var magic = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf);
|
||
if (magic != Magic)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
var version = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[4..]);
|
||
if (version < MinSupportedVersion || version > FormatVersion)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
var mapId = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[8..]);
|
||
var fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[12..]);
|
||
var bakeTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[20..]);
|
||
var chunkCount = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[28..]);
|
||
var indexOffset = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[32..]);
|
||
|
||
if (fingerprint != ComputeFingerprint((int)mapId))
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
// Pull the whole index in one read.
|
||
var indexBytes = (int)chunkCount * IndexEntryBytes;
|
||
var indexBuf = new byte[indexBytes];
|
||
stream.Position = (long)indexOffset;
|
||
if (stream.Read(indexBuf, 0, indexBytes) != indexBytes)
|
||
{
|
||
stream.Dispose();
|
||
return null;
|
||
}
|
||
|
||
// Entries are in record write order and carry no offset, so rebuild each one as a
|
||
// running sum of the record lengths, starting just past the header.
|
||
var offsets = new Dictionary<ulong, (ulong offset, uint length)>((int)chunkCount);
|
||
var runningOffset = (ulong)HeaderSize;
|
||
for (var i = 0; i < chunkCount; i++)
|
||
{
|
||
var entry = indexBuf.AsSpan(i * IndexEntryBytes);
|
||
var packedKey = BinaryPrimitives.ReadUInt32LittleEndian(entry);
|
||
var len = BinaryPrimitives.ReadUInt32LittleEndian(entry[4..]);
|
||
var key = PackChunkKey((int)(packedKey >> 16), (int)(packedKey & 0xFFFF));
|
||
offsets[key] = (runningOffset, len);
|
||
runningOffset += len;
|
||
}
|
||
|
||
return new LazyReader(stream, mapId, fingerprint, bakeTimestamp, chunkCount, offsets);
|
||
}
|
||
catch
|
||
{
|
||
stream?.Dispose();
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private static ulong PackChunkKey(int chunkX, int chunkY) => ((ulong)(uint)chunkX << 32) | (uint)chunkY;
|
||
|
||
/// <summary>
|
||
/// Guesses a cell's destination Z for one direction: on flat ground a step lands at the Z you
|
||
/// left from, so predict SourceZ where the direction is passable and 0 where it isn't. The
|
||
/// zero matches the baker, which only writes a slot on a successful step and leaves the rest
|
||
/// cleared. Most terrain is flat, so most predictions are exact and most residuals are 0 —
|
||
/// which is what makes the residual arrays compress away to nothing.
|
||
/// </summary>
|
||
internal static sbyte Predict(byte dirMaskByte, int bit, sbyte sourceZ) =>
|
||
(dirMaskByte >> bit & 1) != 0 ? sourceZ : (sbyte)0;
|
||
|
||
/// <summary>
|
||
/// A destination Z's difference from its prediction. Wraps deliberately: two's-complement
|
||
/// round-trips exactly for every sbyte input, so no value range is off-limits.
|
||
/// </summary>
|
||
internal static sbyte EncodeResidual(sbyte z, sbyte predict) => unchecked((sbyte)(z - predict));
|
||
|
||
/// <summary>Inverse of <see cref="EncodeResidual"/>.</summary>
|
||
internal static sbyte DecodeZ(sbyte predict, sbyte residual) => unchecked((sbyte)(predict + residual));
|
||
|
||
/// <summary>
|
||
/// The destination-Z array for direction index d, in the canonical order the format stores them:
|
||
/// walk N..NW as 0-7, then swim N..NW as 8-15.
|
||
/// </summary>
|
||
private static sbyte[] GetBaseZArray(StepChunk c, int d) => d switch
|
||
{
|
||
0 => c.WalkZN, 1 => c.WalkZNE, 2 => c.WalkZE, 3 => c.WalkZSE,
|
||
4 => c.WalkZS, 5 => c.WalkZSW, 6 => c.WalkZW, 7 => c.WalkZNW,
|
||
8 => c.SwimZN, 9 => c.SwimZNE, 10 => c.SwimZE, 11 => c.SwimZSE,
|
||
12 => c.SwimZS, 13 => c.SwimZSW, 14 => c.SwimZW, 15 => c.SwimZNW,
|
||
_ => throw new ArgumentOutOfRangeException(nameof(d))
|
||
};
|
||
|
||
/// <summary>
|
||
/// Builds one chunk's record, compresses it, and frames it as [u32 uncompressedLen][payload].
|
||
/// When compression fails to shrink the record — as it does on the tiny Uniform ones — the raw
|
||
/// record is stored instead, and the reader tells the two apart by payload length.
|
||
/// </summary>
|
||
private static void WriteChunk(
|
||
BufferWriter w, int chunkX, int chunkY, StepChunk chunk,
|
||
LibDeflateBinding packer, ref byte[] recordScratch, ref byte[] compScratch
|
||
)
|
||
{
|
||
var rw = new BufferWriter(recordScratch, prefixStr: false);
|
||
BuildRecord(rw, chunkX, chunkY, chunk);
|
||
recordScratch = rw.Buffer; // may have grown; hold onto the larger buffer for the next chunk
|
||
var recordLen = (int)rw.Position;
|
||
|
||
var bound = packer.MaxPackSize(recordLen);
|
||
if (compScratch.Length < bound)
|
||
{
|
||
compScratch = new byte[bound];
|
||
}
|
||
|
||
var compLen = packer.Pack(compScratch, recordScratch.AsSpan(0, recordLen));
|
||
|
||
w.Write((uint)recordLen);
|
||
if (compLen > 0 && compLen < recordLen)
|
||
{
|
||
w.Write(compScratch.AsSpan(0, compLen));
|
||
}
|
||
else
|
||
{
|
||
// Compression didn't help, so store the record raw. Payload length == uncompressedLen
|
||
// is how the reader recognizes that.
|
||
w.Write(recordScratch.AsSpan(0, recordLen));
|
||
}
|
||
}
|
||
|
||
private static void BuildRecord(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
|
||
{
|
||
w.Write((ushort)chunkX);
|
||
w.Write((ushort)chunkY);
|
||
w.Write((uint)chunk.BuiltMultisVersion);
|
||
|
||
// A uniform chunk — every cell identical — collapses to one cell's worth of data, ~28 bytes.
|
||
// Open water and solid rock make up a lot of a map, so this is worth the branch.
|
||
if (chunk.IsUniform())
|
||
{
|
||
w.Write(KindUniform);
|
||
w.Write(chunk.WalkMask[0]);
|
||
w.Write(chunk.WetMask[0]);
|
||
w.Write((byte)chunk.SourceZ[0]);
|
||
w.Write((byte)chunk.WalkZN[0]);
|
||
w.Write((byte)chunk.WalkZNE[0]);
|
||
w.Write((byte)chunk.WalkZE[0]);
|
||
w.Write((byte)chunk.WalkZSE[0]);
|
||
w.Write((byte)chunk.WalkZS[0]);
|
||
w.Write((byte)chunk.WalkZSW[0]);
|
||
w.Write((byte)chunk.WalkZW[0]);
|
||
w.Write((byte)chunk.WalkZNW[0]);
|
||
w.Write((byte)chunk.SwimZN[0]);
|
||
w.Write((byte)chunk.SwimZNE[0]);
|
||
w.Write((byte)chunk.SwimZE[0]);
|
||
w.Write((byte)chunk.SwimZSE[0]);
|
||
w.Write((byte)chunk.SwimZS[0]);
|
||
w.Write((byte)chunk.SwimZSW[0]);
|
||
w.Write((byte)chunk.SwimZW[0]);
|
||
w.Write((byte)chunk.SwimZNW[0]);
|
||
return;
|
||
}
|
||
|
||
w.Write(KindFull);
|
||
|
||
var strataOffsetByCell = chunk.GetStrataOffsetByCellForSerialization();
|
||
var strataData = chunk.GetStrataDataForSerialization();
|
||
var hasStrata = strataOffsetByCell != null;
|
||
var hasSwimLayer = chunk.HasSwimLayer;
|
||
w.Write((byte)(hasStrata ? 1 : 0));
|
||
w.Write((byte)(hasSwimLayer ? 1 : 0));
|
||
|
||
// Each destination-Z array is stored as residuals against its prediction (see Predict). An
|
||
// array that matches its prediction everywhere — the common case on flat terrain — is
|
||
// omitted entirely, and its ZArrayMask bit stays clear so the reader synthesizes it.
|
||
ushort zArrayMask = 0;
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
var z = GetBaseZArray(chunk, d);
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
if (z[cell] != Predict(dirMask[cell], bit, chunk.SourceZ[cell]))
|
||
{
|
||
zArrayMask |= (ushort)(1 << d);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
w.Write(zArrayMask);
|
||
|
||
w.Write(chunk.WalkMask);
|
||
w.Write(chunk.WetMask);
|
||
WriteSBytes(w, chunk.SourceZ);
|
||
|
||
Span<sbyte> residual = stackalloc sbyte[StepChunk.CellsPerChunk];
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
if ((zArrayMask >> d & 1) == 0)
|
||
{
|
||
continue;
|
||
}
|
||
var z = GetBaseZArray(chunk, d);
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
residual[cell] = EncodeResidual(z[cell], Predict(dirMask[cell], bit, chunk.SourceZ[cell]));
|
||
}
|
||
w.Write(MemoryMarshal.Cast<sbyte, byte>(residual));
|
||
}
|
||
|
||
if (hasSwimLayer)
|
||
{
|
||
WriteSBytes(w, chunk.SwimSourceZ);
|
||
w.Write(chunk.SwimMask);
|
||
WriteSBytes(w, chunk.SwimZN_Layer);
|
||
WriteSBytes(w, chunk.SwimZNE_Layer);
|
||
WriteSBytes(w, chunk.SwimZE_Layer);
|
||
WriteSBytes(w, chunk.SwimZSE_Layer);
|
||
WriteSBytes(w, chunk.SwimZS_Layer);
|
||
WriteSBytes(w, chunk.SwimZSW_Layer);
|
||
WriteSBytes(w, chunk.SwimZW_Layer);
|
||
WriteSBytes(w, chunk.SwimZNW_Layer);
|
||
}
|
||
|
||
if (hasStrata)
|
||
{
|
||
for (var i = 0; i < StepChunk.CellsPerChunk; i++)
|
||
{
|
||
w.Write(strataOffsetByCell[i]);
|
||
}
|
||
var dataLen = (uint)(strataData?.Length ?? 0);
|
||
w.Write(dataLen);
|
||
if (dataLen > 0)
|
||
{
|
||
w.Write(strataData);
|
||
}
|
||
}
|
||
}
|
||
|
||
private static StepChunk ReadChunk(byte[] buffer)
|
||
{
|
||
var r = new BufferReader(buffer);
|
||
// ChunkX + ChunkY — already known from the index lookup that got us here.
|
||
r.ReadUShort();
|
||
r.ReadUShort();
|
||
var multisVersion = (int)r.ReadUInt();
|
||
var kind = r.ReadByte();
|
||
|
||
var chunk = new StepChunk { BuiltMultisVersion = multisVersion };
|
||
|
||
if (kind == KindUniform) // one cell's values, broadcast to all 256
|
||
{
|
||
Array.Fill(chunk.WalkMask, r.ReadByte());
|
||
Array.Fill(chunk.WetMask, r.ReadByte());
|
||
Array.Fill(chunk.SourceZ, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZN, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZNE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZSE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZS, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZSW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.WalkZNW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZN, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZNE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZSE, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZS, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZSW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZW, (sbyte)r.ReadByte());
|
||
Array.Fill(chunk.SwimZNW, (sbyte)r.ReadByte());
|
||
return chunk;
|
||
}
|
||
|
||
var hasStrata = r.ReadByte() != 0;
|
||
var hasSwimLayer = r.ReadByte() != 0;
|
||
var zArrayMask = r.ReadUShort();
|
||
|
||
r.Read(chunk.WalkMask);
|
||
r.Read(chunk.WetMask);
|
||
ReadSBytes(r, chunk.SourceZ);
|
||
|
||
// Inverse of the write path: a stored array carries residuals to add back to the
|
||
// prediction, an omitted one IS the prediction.
|
||
Span<sbyte> residual = stackalloc sbyte[StepChunk.CellsPerChunk];
|
||
for (var d = 0; d < 16; d++)
|
||
{
|
||
var z = GetBaseZArray(chunk, d);
|
||
var dirMask = d < 8 ? chunk.WalkMask : chunk.WetMask;
|
||
var bit = d & 7;
|
||
if ((zArrayMask >> d & 1) != 0)
|
||
{
|
||
r.Read(MemoryMarshal.Cast<sbyte, byte>(residual));
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
z[cell] = DecodeZ(Predict(dirMask[cell], bit, chunk.SourceZ[cell]), residual[cell]);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
for (var cell = 0; cell < StepChunk.CellsPerChunk; cell++)
|
||
{
|
||
z[cell] = Predict(dirMask[cell], bit, chunk.SourceZ[cell]);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (hasSwimLayer)
|
||
{
|
||
chunk.AllocateSwimLayer();
|
||
ReadSBytes(r, chunk.SwimSourceZ);
|
||
r.Read(chunk.SwimMask);
|
||
ReadSBytes(r, chunk.SwimZN_Layer);
|
||
ReadSBytes(r, chunk.SwimZNE_Layer);
|
||
ReadSBytes(r, chunk.SwimZE_Layer);
|
||
ReadSBytes(r, chunk.SwimZSE_Layer);
|
||
ReadSBytes(r, chunk.SwimZS_Layer);
|
||
ReadSBytes(r, chunk.SwimZSW_Layer);
|
||
ReadSBytes(r, chunk.SwimZW_Layer);
|
||
ReadSBytes(r, chunk.SwimZNW_Layer);
|
||
}
|
||
|
||
if (hasStrata)
|
||
{
|
||
var offsets = new ushort[StepChunk.CellsPerChunk];
|
||
for (var i = 0; i < offsets.Length; i++)
|
||
{
|
||
offsets[i] = r.ReadUShort();
|
||
}
|
||
var dataLen = (int)r.ReadUInt();
|
||
var data = new byte[dataLen];
|
||
if (dataLen > 0)
|
||
{
|
||
r.Read(data);
|
||
}
|
||
chunk.SetStrata(offsets, data);
|
||
}
|
||
|
||
return chunk;
|
||
}
|
||
|
||
private static void WriteSBytes(BufferWriter w, sbyte[] arr) =>
|
||
w.Write(MemoryMarshal.Cast<sbyte, byte>(arr.AsSpan()));
|
||
|
||
private static void ReadSBytes(BufferReader r, sbyte[] arr) =>
|
||
r.Read(MemoryMarshal.Cast<sbyte, byte>(arr.AsSpan()));
|
||
|
||
/// <summary>
|
||
/// An open .swb file: the stream plus the chunk index. Only the records actually asked for are
|
||
/// ever read or inflated. Dispose releases the stream.
|
||
/// </summary>
|
||
internal sealed class LazyReader : IDisposable
|
||
{
|
||
private FileStream _stream;
|
||
private readonly Dictionary<ulong, (ulong offset, uint length)> _offsets;
|
||
private byte[] _buffer; // the raw framed record as it sits on disk
|
||
private byte[] _bodyBuffer; // that record, inflated, ready for ReadChunk
|
||
|
||
public uint MapId { get; }
|
||
public ulong Fingerprint { get; }
|
||
public ulong BakeTimestamp { get; }
|
||
public uint ChunkCount { get; }
|
||
public int IndexedChunkCount => _offsets.Count;
|
||
|
||
public bool Has(int chunkX, int chunkY) => _offsets.ContainsKey(PackChunkKey(chunkX, chunkY));
|
||
|
||
/// <summary>Every (chunkX, chunkY) the file holds. Used to preload the whole file.</summary>
|
||
public IEnumerable<(int chunkX, int chunkY)> EnumerateChunkCoords()
|
||
{
|
||
foreach (var key in _offsets.Keys)
|
||
{
|
||
yield return ((int)(key >> 32), (int)(key & 0xFFFFFFFF));
|
||
}
|
||
}
|
||
|
||
internal LazyReader(
|
||
FileStream stream, uint mapId, ulong fingerprint, ulong bakeTimestamp,
|
||
uint chunkCount, Dictionary<ulong, (ulong offset, uint length)> offsets
|
||
)
|
||
{
|
||
_stream = stream;
|
||
MapId = mapId;
|
||
Fingerprint = fingerprint;
|
||
BakeTimestamp = bakeTimestamp;
|
||
ChunkCount = chunkCount;
|
||
_offsets = offsets;
|
||
_buffer = new byte[BytesPerChunkBase];
|
||
_bodyBuffer = new byte[BytesPerChunkBase];
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads one chunk from the file, or null if the file has no record for it. One seek and
|
||
/// one read, sized to the record's indexed length.
|
||
/// </summary>
|
||
public StepChunk TryReadChunk(int chunkX, int chunkY)
|
||
{
|
||
if (_stream == null)
|
||
{
|
||
return null;
|
||
}
|
||
|
||
var key = PackChunkKey(chunkX, chunkY);
|
||
if (!_offsets.TryGetValue(key, out var entry))
|
||
{
|
||
return null;
|
||
}
|
||
|
||
if (entry.length > _buffer.Length)
|
||
{
|
||
_buffer = new byte[entry.length];
|
||
}
|
||
|
||
_stream.Position = (long)entry.offset;
|
||
var read = _stream.Read(_buffer, 0, (int)entry.length);
|
||
if (read < (int)entry.length || entry.length < sizeof(uint))
|
||
{
|
||
return null;
|
||
}
|
||
|
||
// [u32 uncompressedLen][payload], where the payload is compressed unless its length
|
||
// already equals uncompressedLen — then it was stored raw.
|
||
var uncompressedLen = (int)BinaryPrimitives.ReadUInt32LittleEndian(_buffer);
|
||
var payloadLen = (int)entry.length - sizeof(uint);
|
||
if (_bodyBuffer.Length < uncompressedLen)
|
||
{
|
||
_bodyBuffer = new byte[uncompressedLen];
|
||
}
|
||
|
||
if (payloadLen == uncompressedLen)
|
||
{
|
||
Array.Copy(_buffer, sizeof(uint), _bodyBuffer, 0, uncompressedLen);
|
||
}
|
||
else
|
||
{
|
||
// Deflate.Standard, not .Maximum: the level only affects packing, and inflate has
|
||
// to accept whatever the writer produced regardless.
|
||
var result = Deflate.Standard.Unpack(
|
||
_bodyBuffer.AsSpan(0, uncompressedLen),
|
||
_buffer.AsSpan(sizeof(uint), payloadLen),
|
||
out var produced
|
||
);
|
||
if (result != LibDeflateResult.Success || produced != uncompressedLen)
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
return ReadChunk(_bodyBuffer);
|
||
}
|
||
|
||
public void Dispose()
|
||
{
|
||
_stream?.Dispose();
|
||
_stream = null;
|
||
_buffer = null;
|
||
_bodyBuffer = null;
|
||
}
|
||
}
|
||
}
|