ModernUO/Projects/UOContent/Engines/Pathing/Cache/StepCacheFile.cs

using System;
using System.Buffers.Binary;
using System.Collections.Generic;
using System.IO;
using System.Runtime.InteropServices;

namespace Server.Engines.Pathing.Cache;

/// <summary>
/// Binary serializer + lazy reader for the step cache. Persists chunk records to disk
/// so a server warm-starts without paying chunk-build cost on the first pathfind through
/// a region. Lazy: opening a file reads only the header + chunk-offset index (~few KB
/// for tens of thousands of chunks), then individual chunks are seeked + deserialized
/// only when the cache asks for them. RAM stays bounded by MaxResidentChunks regardless
/// of file size.
///
/// File layout (little-endian, BufferWriter / BufferReader convention):
///
///   Header (48 bytes):
///     u32   Magic           = 0x42575300 ('SWB\0')
///     u32   Version         = current FormatVersion
///     u32   MapId
///     u64   Fingerprint     XxHash3 over (1) LandTable + ItemTable flags AND (2) the
///                           on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul.
///                           Rejects a load when EITHER tile flags shifted (client patch)
///                           OR the map data was rewritten (CentredSharp / UOFiddler edit).
///                           The .mul format has no built-in CRC; this is the only way
///                           to detect those mutations.
///     u64   BakeTimestamp   DateTime.UtcNow.Ticks at write time (informational).
///     u32   ChunkCount
///     u64   IndexOffset     File position where the chunk index begins.
///
///   Per chunk (ChunkCount times, variable size):
///     u16   ChunkX
///     u16   ChunkY
///     u32   BuiltMultisVersion
///     u8    HasMultiZ       0 = no MultiZCells follow; 1 = 32 bytes of MultiZCells follow
///     byte  WalkMask[256]
///     byte  WetMask[256]
///     sbyte SourceZ[256]
///     sbyte WalkZN[256]..WalkZNW[256]    (8 arrays in N,NE,E,SE,S,SW,W,NW order)
///     sbyte SwimZN[256]..SwimZNW[256]    (8 arrays in same order)
///     [byte MultiZCells[32] — only when HasMultiZ == 1]
///
///   Index trailer (16 × ChunkCount bytes):
///     For each chunk: { u64 chunkKey, u64 fileOffset }
///
/// Per-chunk size: ~5,393 bytes (no multi-Z) or ~5,425 bytes (with multi-Z).
/// LRU bookkeeping (LastTouchedTicks) is intentionally not persisted.
/// </summary>
internal static class StepCacheFile
{
    public const uint Magic = 0x42575300; // 'SWB\0'
    public const uint FormatVersion = 1;

    private const int HeaderSize =
        sizeof(uint)    // Magic
        + sizeof(uint)  // Version
        + sizeof(uint)  // MapId
        + sizeof(ulong) // Fingerprint
        + sizeof(ulong) // BakeTimestamp
        + sizeof(uint)  // ChunkCount
        + sizeof(ulong); // IndexOffset

    private const int IndexEntryBytes = sizeof(ulong) + sizeof(ulong); // chunkKey + offset

    private const int BytesPerChunkBase =
        sizeof(ushort) + sizeof(ushort) + sizeof(uint) + sizeof(byte)
        + StepChunk.CellsPerChunk           // WalkMask
        + StepChunk.CellsPerChunk           // WetMask
        + StepChunk.CellsPerChunk           // SourceZ
        + 8 * StepChunk.CellsPerChunk       // WalkZ[8]
        + 8 * StepChunk.CellsPerChunk;      // SwimZ[8]

    private const int BytesPerMultiZ = 32;

    /// <summary>
    /// Byte offset of the IndexOffset u64 within the header
    /// (Magic+Version+MapId+Fingerprint+BakeTimestamp+ChunkCount = 32). Patched after chunks land.
    /// </summary>
    private const int IndexOffsetFieldPosition = 32;

    public delegate bool ChunkEnumerator(out int chunkX, out int chunkY, out StepChunk chunk);

    /// <summary>
    /// Peek at a .swb file's Fingerprint field (header byte offset 12) without
    /// reading any chunk data. Returns false on missing file, bad magic, or wrong
    /// version. Cheap — reads 20 bytes total.
    /// </summary>
    public static bool TryReadFingerprint(string path, out ulong fingerprint)
    {
        fingerprint = 0;
        if (!File.Exists(path))
        {
            return false;
        }

        try
        {
            using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
            Span<byte> buf = stackalloc byte[20];
            if (stream.Read(buf) < 20)
            {
                return false;
            }
            if (BinaryPrimitives.ReadUInt32LittleEndian(buf) != Magic)
            {
                return false;
            }
            if (BinaryPrimitives.ReadUInt32LittleEndian(buf[4..]) != FormatVersion)
            {
                return false;
            }
            // mapId is at buf[8..12], we skip; hash is at buf[12..20].
            fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(buf[12..]);
            return true;
        }
        catch
        {
            return false;
        }
    }

    /// <summary>
    /// Combined XxHash3 fingerprint over (1) the loaded TileData flag tables and (2) the
    /// per-map .mul / .uop file contents (via <see cref="TileMatrix.MapFilesFingerprint"/>).
    /// Bake files carry this hash so a load can refuse to populate the cache when EITHER
    /// tile flags shifted (client patch) OR the map data was rewritten (CentredSharp /
    /// UOFiddler edit). The .mul format has no built-in CRC; this is the only way to
    /// detect those mutations.
    /// </summary>
    public static ulong ComputeFingerprint(int mapId)
    {
        var hasher = HashUtility.CreateXxHash3();

        // TileData flag tables — same projection trick as before: just the Flags ulong
        // from each entry, written little-endian into a contiguous byte buffer. The
        // struct itself has a string Name (reference) whose object identity isn't
        // stable across runs, so MemoryMarshal.Cast over the whole struct would drift.
        var landTable = TileData.LandTable;
        var itemTable = TileData.ItemTable;
        var bytes = new byte[(landTable.Length + itemTable.Length) * sizeof(ulong)];
        var span = bytes.AsSpan();

        for (var i = 0; i < landTable.Length; i++)
        {
            BinaryPrimitives.WriteUInt64LittleEndian(span[(i * 8)..], (ulong)landTable[i].Flags);
        }
        var itemOffset = landTable.Length * 8;
        for (var i = 0; i < itemTable.Length; i++)
        {
            BinaryPrimitives.WriteUInt64LittleEndian(span[(itemOffset + i * 8)..], (ulong)itemTable[i].Flags);
        }
        hasher.Append(bytes);

        // Map files (mapX.mul / .uop, staidxX.mul, staticsX.mul). TileMatrix already
        // streamed them through XxHash3 once at construction; mix the result in.
        var map = Map.Maps[mapId];
        if (map != null && map != Map.Internal && map.Tiles != null)
        {
            Span<byte> mapHashBytes = stackalloc byte[sizeof(ulong)];
            BinaryPrimitives.WriteUInt64LittleEndian(mapHashBytes, map.Tiles.MapFilesFingerprint);
            hasher.Append(mapHashBytes);
        }

        return hasher.GetCurrentHashAsUInt64();
    }

    /// <summary>
    /// Writes the file: header (with placeholder IndexOffset) → chunks (offsets recorded)
    /// → index trailer → patches the header IndexOffset. <paramref name="chunkCount"/> must
    /// equal the actual number of chunks <paramref name="next"/> will yield.
    /// </summary>
    public static void Write(string path, uint mapId, uint chunkCount, ChunkEnumerator next)
    {
        Directory.CreateDirectory(Path.GetDirectoryName(path) ?? ".");

        var capacity = HeaderSize
                       + (BytesPerChunkBase + BytesPerMultiZ) * (int)chunkCount
                       + IndexEntryBytes * (int)chunkCount;
        var buffer = new byte[capacity];
        var w = new BufferWriter(buffer, prefixStr: false);

        w.Write(Magic);
        w.Write(FormatVersion);
        w.Write(mapId);
        w.Write(ComputeFingerprint((int)mapId));
        w.Write((ulong)DateTime.UtcNow.Ticks);
        w.Write(chunkCount);
        w.Write(0UL); // IndexOffset placeholder, patched after chunks

        var indexEntries = new (ulong key, ulong offset)[chunkCount];
        var written = 0u;
        while (next(out var chunkX, out var chunkY, out var chunk))
        {
            if (written >= chunkCount)
            {
                throw new InvalidOperationException(
                    $"StepCacheFile.Write: enumerator yielded more than the declared {chunkCount} chunks"
                );
            }
            var chunkOffset = (ulong)w.Position;
            WriteChunk(w, chunkX, chunkY, chunk);
            indexEntries[written] = (PackChunkKey(chunkX, chunkY), chunkOffset);
            written++;
        }

        if (written != chunkCount)
        {
            throw new InvalidOperationException(
                $"StepCacheFile.Write: declared {chunkCount} chunks but enumerator yielded {written}"
            );
        }

        var indexOffset = (ulong)w.Position;
        for (var i = 0u; i < chunkCount; i++)
        {
            w.Write(indexEntries[i].key);
            w.Write(indexEntries[i].offset);
        }

        // Patch IndexOffset directly into the buffer (BufferWriter has no Seek).
        BinaryPrimitives.WriteUInt64LittleEndian(buffer.AsSpan(IndexOffsetFieldPosition, 8), indexOffset);

        var totalBytes = (int)w.Position;
        File.WriteAllBytes(path, buffer.AsSpan(0, totalBytes).ToArray());
    }

    /// <summary>
    /// Opens a .swb file and reads only its header + chunk-offset index. Returns null on
    /// missing file, magic / version mismatch, or Fingerprint mismatch (a stale bake
    /// against a freshly patched client). Callers own disposal of the returned reader.
    /// </summary>
    public static LazyReader OpenForLazy(string path)
    {
        if (!File.Exists(path))
        {
            return null;
        }

        FileStream stream = null;
        try
        {
            stream = new FileStream(
                path,
                FileMode.Open,
                FileAccess.Read,
                FileShare.Read | FileShare.Delete
            );

            Span<byte> headerBuf = stackalloc byte[HeaderSize];
            if (stream.Read(headerBuf) != HeaderSize)
            {
                stream.Dispose();
                return null;
            }

            var magic = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf);
            if (magic != Magic)
            {
                stream.Dispose();
                return null;
            }
            var version = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[4..]);
            if (version != FormatVersion)
            {
                stream.Dispose();
                return null;
            }

            var mapId = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[8..]);
            var fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[12..]);
            var bakeTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[20..]);
            var chunkCount = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[28..]);
            var indexOffset = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[32..]);

            if (fingerprint != ComputeFingerprint((int)mapId))
            {
                stream.Dispose();
                return null;
            }

            // Read the chunk-offset index in one shot.
            var indexBytes = (int)chunkCount * IndexEntryBytes;
            var indexBuf = new byte[indexBytes];
            stream.Position = (long)indexOffset;
            if (stream.Read(indexBuf, 0, indexBytes) != indexBytes)
            {
                stream.Dispose();
                return null;
            }

            var offsets = new Dictionary<ulong, ulong>((int)chunkCount);
            for (var i = 0; i < chunkCount; i++)
            {
                var entry = indexBuf.AsSpan(i * IndexEntryBytes);
                var key = BinaryPrimitives.ReadUInt64LittleEndian(entry);
                var off = BinaryPrimitives.ReadUInt64LittleEndian(entry[8..]);
                offsets[key] = off;
            }

            return new LazyReader(stream, mapId, fingerprint, bakeTimestamp, chunkCount, offsets);
        }
        catch
        {
            stream?.Dispose();
            return null;
        }
    }

    private static ulong PackChunkKey(int chunkX, int chunkY) => ((ulong)(uint)chunkX << 32) | (uint)chunkY;

    private static void WriteChunk(BufferWriter w, int chunkX, int chunkY, StepChunk chunk)
    {
        w.Write((ushort)chunkX);
        w.Write((ushort)chunkY);
        w.Write((uint)chunk.BuiltMultisVersion);

        var multiZ = chunk.GetMultiZCellsForSerialization();
        w.Write((byte)(multiZ != null ? 1 : 0));

        w.Write(chunk.WalkMask);
        w.Write(chunk.WetMask);
        WriteSBytes(w, chunk.SourceZ);

        WriteSBytes(w, chunk.WalkZN);
        WriteSBytes(w, chunk.WalkZNE);
        WriteSBytes(w, chunk.WalkZE);
        WriteSBytes(w, chunk.WalkZSE);
        WriteSBytes(w, chunk.WalkZS);
        WriteSBytes(w, chunk.WalkZSW);
        WriteSBytes(w, chunk.WalkZW);
        WriteSBytes(w, chunk.WalkZNW);

        WriteSBytes(w, chunk.SwimZN);
        WriteSBytes(w, chunk.SwimZNE);
        WriteSBytes(w, chunk.SwimZE);
        WriteSBytes(w, chunk.SwimZSE);
        WriteSBytes(w, chunk.SwimZS);
        WriteSBytes(w, chunk.SwimZSW);
        WriteSBytes(w, chunk.SwimZW);
        WriteSBytes(w, chunk.SwimZNW);

        if (multiZ != null)
        {
            w.Write(multiZ);
        }
    }

    private static StepChunk ReadChunk(byte[] buffer)
    {
        var r = new BufferReader(buffer);
        // Skip ChunkX + ChunkY (already known via the index lookup).
        r.ReadUShort();
        r.ReadUShort();
        var multisVersion = (int)r.ReadUInt();
        var hasMultiZ = r.ReadByte() != 0;

        var chunk = new StepChunk { BuiltMultisVersion = multisVersion };

        r.Read(chunk.WalkMask);
        r.Read(chunk.WetMask);
        ReadSBytes(r, chunk.SourceZ);

        ReadSBytes(r, chunk.WalkZN);
        ReadSBytes(r, chunk.WalkZNE);
        ReadSBytes(r, chunk.WalkZE);
        ReadSBytes(r, chunk.WalkZSE);
        ReadSBytes(r, chunk.WalkZS);
        ReadSBytes(r, chunk.WalkZSW);
        ReadSBytes(r, chunk.WalkZW);
        ReadSBytes(r, chunk.WalkZNW);

        ReadSBytes(r, chunk.SwimZN);
        ReadSBytes(r, chunk.SwimZNE);
        ReadSBytes(r, chunk.SwimZE);
        ReadSBytes(r, chunk.SwimZSE);
        ReadSBytes(r, chunk.SwimZS);
        ReadSBytes(r, chunk.SwimZSW);
        ReadSBytes(r, chunk.SwimZW);
        ReadSBytes(r, chunk.SwimZNW);

        if (hasMultiZ)
        {
            var multiZ = new byte[BytesPerMultiZ];
            r.Read(multiZ);
            chunk.RestoreMultiZCellsFromSerialization(multiZ);
        }

        return chunk;
    }

    private static void WriteSBytes(BufferWriter w, sbyte[] arr) =>
        w.Write(MemoryMarshal.Cast<sbyte, byte>(arr.AsSpan()));

    private static void ReadSBytes(BufferReader r, sbyte[] arr) =>
        r.Read(MemoryMarshal.Cast<sbyte, byte>(arr.AsSpan()));

    /// <summary>
    /// Open handle on a .swb file. Holds the FileStream + chunk-offset index. Chunks are
    /// fetched on demand via <see cref="TryReadChunk"/>; only the records actually queried
    /// are ever materialized. Dispose releases the underlying stream.
    /// </summary>
    internal sealed class LazyReader : IDisposable
    {
        private FileStream _stream;
        private readonly Dictionary<ulong, ulong> _offsets;
        private byte[] _buffer;

        public uint MapId { get; }
        public ulong Fingerprint { get; }
        public ulong BakeTimestamp { get; }
        public uint ChunkCount { get; }
        public int IndexedChunkCount => _offsets.Count;

        public bool Has(int chunkX, int chunkY) => _offsets.ContainsKey(PackChunkKey(chunkX, chunkY));

        internal LazyReader(
            FileStream stream, uint mapId, ulong fingerprint, ulong bakeTimestamp,
            uint chunkCount, Dictionary<ulong, ulong> offsets
        )
        {
            _stream = stream;
            MapId = mapId;
            Fingerprint = fingerprint;
            BakeTimestamp = bakeTimestamp;
            ChunkCount = chunkCount;
            _offsets = offsets;
            _buffer = new byte[BytesPerChunkBase + BytesPerMultiZ];
        }

        /// <summary>
        /// Returns the chunk record at (<paramref name="chunkX"/>, <paramref name="chunkY"/>)
        /// from the file, or null if the file doesn't contain it. Single seek + bulk read;
        /// no allocations beyond the returned StepChunk and its arrays.
        /// </summary>
        public StepChunk TryReadChunk(int chunkX, int chunkY)
        {
            if (_stream == null)
            {
                return null;
            }

            var key = PackChunkKey(chunkX, chunkY);
            if (!_offsets.TryGetValue(key, out var offset))
            {
                return null;
            }

            _stream.Position = (long)offset;
            // Try to read the maximum size; the file may have less remaining, which is OK
            // since BufferReader stops at the bytes it actually needs.
            var read = _stream.Read(_buffer, 0, _buffer.Length);
            return read < BytesPerChunkBase ? null : ReadChunk(_buffer);
        }

        public void Dispose()
        {
            _stream?.Dispose();
            _stream = null;
            _buffer = null;
        }
    }
}