feat(pathfinding): JSONL recorder + public bake helpers (#2449)
## Summary Adds two pieces of pathfinding tooling on top of PR #2448's lazy `.swb` infrastructure: - **`PathfindRecorder`** — admin-toggled JSONL telemetry capture; one record per `BitmapAStarAlgorithm.Find` call. Output format matches the BDN harness corpus, so production traffic can be captured and replayed in benchmarks without an adapter. - **Public bake helpers on `StepCache`** — `ComputeLiveTileDataHash`, `TryReadTileDataHashFromFile`, `BakeMap`, `ClearResidentChunks`. Lets the benchmark project (and any future bake utility) drive cache fill + persist without exposing internal types. The companion BDN harness update lives in [ModernUO-Benchmarks#kb/pathfinding-pr4-bench](https://github.com/modernuo/ModernUO-Benchmarks/tree/kb/pathfinding-pr4-bench): porting `Benchmarks/PathfindInGame/` from the `kb/ai_pathfinding` branch to the API shipped in #2446–#2448. ## What's in this PR ### `PathfindRecorder` (`PathfindRecorder.cs`) - Holds a single `StreamWriter` open while recording; its internal buffer absorbs per-record writes without per-call `File.AppendAllText`. - Single `bool` check on the hot path; cheap when disabled. - Disabling flushes + disposes; an IO failure during write also disables the recorder. - Server config: - `pathfinding.recorder.enable` — bool, default `false`. Read on boot via `GetOrUpdateSetting`. - `pathfinding.recorder.path` — default `<basedir>/Data/Pathfinding/recordings/pathfinds.jsonl`. - Hooked into `BitmapAStarAlgorithm.Find` — runs once per call, does nothing when disabled. - Admin command: `[PathRecord [on|off|flush|status]` (default `status`). ### Public cache helpers - `static ulong StepCache.ComputeLiveTileDataHash()` — wraps the file module's hash function for staleness checks. - `static bool StepCache.TryReadTileDataHashFromFile(string, out ulong)` — peeks at a `.swb` file's hash field (20 bytes). - `int StepCache.BakeMap(int, string)` — walks every chunk in the map, populates resident set, saves. Offline / fixture use; blocks for many seconds on a full-map walk. - `void StepCache.ClearResidentChunks()` — drops chunks + zeros counters but keeps lazy readers open. Lets benchmark loops measure "first query after boot" cost across iterations without the lazy-reader reopen overhead.
This commit is contained in:
parent
7c9215d97c
commit
a8ca82738d
10 changed files with 587 additions and 27 deletions
|
|
@ -17,6 +17,7 @@ using Server.Logging;
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Hashing;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
|
|
@ -48,6 +49,16 @@ public class TileMatrix
|
|||
public FileStream DataStream { get; }
|
||||
public BinaryReader IndexReader { get; }
|
||||
|
||||
/// <summary>
|
||||
/// XxHash3 fingerprint over the bytes of the three on-disk files for this map
|
||||
/// (mapX.mul / .uop, staidxX.mul, staticsX.mul). The .mul format has no built-in
|
||||
/// CRC and CentredSharp / UOFiddler rewrite these files in place, so this is the
|
||||
/// only way to detect "client patched / mapper saved" changes that invalidate
|
||||
/// downstream caches keyed on map data. Computed once at construction; never
|
||||
/// changes for the lifetime of the TileMatrix.
|
||||
/// </summary>
|
||||
public ulong MapFilesFingerprint { get; }
|
||||
|
||||
public static bool Pre6000ClientSupport { get; private set; }
|
||||
|
||||
public static void Configure()
|
||||
|
|
@ -140,6 +151,12 @@ public class TileMatrix
|
|||
{
|
||||
logger.Warning("{File} was not found.", $"statics{fileIndex}.mul");
|
||||
}
|
||||
|
||||
// Stream all three on-disk files through XxHash3 once. Saves + restores
|
||||
// each stream's position so this is invisible to the rest of the constructor
|
||||
// (UOP path has already read part of MapStream by now). Cost is ~5-10ms per
|
||||
// map at boot — paid once, then MapFilesFingerprint is a constant.
|
||||
MapFilesFingerprint = ComputeMapFilesFingerprint();
|
||||
}
|
||||
|
||||
_emptyStaticBlock = new StaticTile[8][][];
|
||||
|
|
@ -164,6 +181,27 @@ public class TileMatrix
|
|||
Patch = new TileMatrixPatch(this, fileIndex);
|
||||
}
|
||||
|
||||
private ulong ComputeMapFilesFingerprint()
|
||||
{
|
||||
var hasher = HashUtility.CreateXxHash3();
|
||||
AppendStreamFromStart(hasher, MapStream);
|
||||
AppendStreamFromStart(hasher, IndexStream);
|
||||
AppendStreamFromStart(hasher, DataStream);
|
||||
return hasher.GetCurrentHashAsUInt64();
|
||||
}
|
||||
|
||||
private static void AppendStreamFromStart(XxHash3 hasher, FileStream stream)
|
||||
{
|
||||
if (stream == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
var originalPosition = stream.Position;
|
||||
stream.Position = 0;
|
||||
hasher.Append(stream);
|
||||
stream.Position = originalPosition;
|
||||
}
|
||||
|
||||
public StaticTile[][][] EmptyStaticBlock => _emptyStaticBlock;
|
||||
|
||||
public void SetStaticBlock(int x, int y, StaticTile[][][] value)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
*************************************************************************/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.IO.Hashing;
|
||||
using System.Numerics;
|
||||
using System.Runtime.InteropServices;
|
||||
|
|
@ -66,6 +67,36 @@ public static class HashUtility
|
|||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One-shot streaming hash. Reads from the stream's current position to its end and
|
||||
/// returns the XxHash3 result. The caller is responsible for setting the stream
|
||||
/// position before the call (and restoring it afterward, if the stream will be reused).
|
||||
/// Returns 0 on a null or unreadable stream.
|
||||
/// </summary>
|
||||
public static ulong ComputeHash64(Stream stream)
|
||||
{
|
||||
if (stream == null || !stream.CanRead)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var hasher = _xxHash3 ??= new XxHash3(unchecked((long)xxHash3Seed));
|
||||
hasher.Append(stream);
|
||||
|
||||
var result = hasher.GetCurrentHashAsUInt64();
|
||||
hasher.Reset();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a fresh <see cref="XxHash3"/> seeded with HashUtility's standard seed.
|
||||
/// Use this when you need to combine multiple inputs into a single hash via
|
||||
/// successive Append calls — the one-shot ComputeHash64 overloads are stateless
|
||||
/// (Reset between callers) and can't compose. Caller owns the returned instance.
|
||||
/// </summary>
|
||||
public static XxHash3 CreateXxHash3() => new(unchecked((long)xxHash3Seed));
|
||||
|
||||
public static uint ComputeHash32(ReadOnlySpan<char> str)
|
||||
{
|
||||
if (str == ReadOnlySpan<char>.Empty)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,163 @@
|
|||
using System.IO;
|
||||
using Server.Engines.Pathing;
|
||||
using Xunit;
|
||||
|
||||
namespace Server.Tests.Pathfinding;
|
||||
|
||||
[Collection("Sequential Pathfinding Tests")]
|
||||
public class PathfindRecorderTests
|
||||
{
|
||||
private static string NewTempPath() =>
|
||||
Path.Combine(Path.GetTempPath(), $"pathfind-recorder-{System.Guid.NewGuid():N}.jsonl");
|
||||
|
||||
/// <summary>
|
||||
/// Reflection-set the static _outputPath without going through Configure (which
|
||||
/// reads from server.cfg) so tests don't poison the project's server.cfg.
|
||||
/// </summary>
|
||||
private static void OverrideOutputPath(string path)
|
||||
{
|
||||
typeof(PathfindRecorder).GetField("_outputPath",
|
||||
System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic)!
|
||||
.SetValue(null, path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Disabled_RecordIfEnabled_DoesNothing()
|
||||
{
|
||||
var path = NewTempPath();
|
||||
OverrideOutputPath(path);
|
||||
PathfindRecorder.SetEnabled(false);
|
||||
|
||||
try
|
||||
{
|
||||
var stub = new RecorderStub(World.NewMobile);
|
||||
stub.DefaultMobileInit();
|
||||
stub.MoveToWorld(new Point3D(1500, 1600, 0), Map.Maps[1]);
|
||||
|
||||
PathfindRecorder.RecordIfEnabled(
|
||||
stub, Map.Maps[1],
|
||||
new Point3D(1500, 1600, 0), new Point3D(1498, 1598, 0)
|
||||
);
|
||||
|
||||
stub.Delete();
|
||||
Assert.False(File.Exists(path), "disabled recorder must not create the output file");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(path)) { File.Delete(path); }
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Enabled_RecordIfEnabled_WritesValidJsonlLine()
|
||||
{
|
||||
var path = NewTempPath();
|
||||
OverrideOutputPath(path);
|
||||
PathfindRecorder.SetEnabled(true);
|
||||
|
||||
try
|
||||
{
|
||||
Assert.True(PathfindRecorder.Enabled);
|
||||
|
||||
var stub = new RecorderStub(World.NewMobile);
|
||||
stub.DefaultMobileInit();
|
||||
stub.MoveToWorld(new Point3D(1500, 1600, 0), Map.Maps[1]);
|
||||
|
||||
PathfindRecorder.RecordIfEnabled(
|
||||
stub, Map.Maps[1],
|
||||
new Point3D(1500, 1600, 5), new Point3D(1498, 1598, 5)
|
||||
);
|
||||
|
||||
// Disable to flush + close before reading.
|
||||
PathfindRecorder.SetEnabled(false);
|
||||
stub.Delete();
|
||||
|
||||
Assert.True(File.Exists(path));
|
||||
var content = File.ReadAllText(path).TrimEnd();
|
||||
Assert.Single(content.Split('\n'));
|
||||
Assert.StartsWith("{\"Name\":\"recorded\"", content);
|
||||
Assert.Contains("\"MapId\":1", content);
|
||||
Assert.Contains("\"StartX\":1500", content);
|
||||
Assert.Contains("\"StartY\":1600", content);
|
||||
Assert.Contains("\"GoalX\":1498", content);
|
||||
Assert.Contains("\"GoalY\":1598", content);
|
||||
Assert.Contains("\"CanSwim\":false", content);
|
||||
Assert.EndsWith("}", content);
|
||||
}
|
||||
finally
|
||||
{
|
||||
PathfindRecorder.SetEnabled(false);
|
||||
if (File.Exists(path)) { File.Delete(path); }
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Enabled_RecordsCapabilityFlagsFromBaseCreature()
|
||||
{
|
||||
var path = NewTempPath();
|
||||
OverrideOutputPath(path);
|
||||
PathfindRecorder.SetEnabled(true);
|
||||
|
||||
try
|
||||
{
|
||||
var stub = new RecorderStub(World.NewMobile);
|
||||
stub.DefaultMobileInit();
|
||||
stub.MoveToWorld(new Point3D(1500, 1600, 0), Map.Maps[1]);
|
||||
stub.CanSwim = true;
|
||||
|
||||
PathfindRecorder.RecordIfEnabled(
|
||||
stub, Map.Maps[1],
|
||||
new Point3D(1500, 1600, 0), new Point3D(1498, 1598, 0)
|
||||
);
|
||||
|
||||
PathfindRecorder.SetEnabled(false);
|
||||
stub.Delete();
|
||||
|
||||
var content = File.ReadAllText(path);
|
||||
Assert.Contains("\"CanSwim\":true", content);
|
||||
Assert.Contains("\"CanFly\":false", content);
|
||||
Assert.Contains("\"CanOpenDoors\":", content); // value depends on BaseCreature defaults
|
||||
Assert.Contains("\"CanMoveOverObstacles\":", content); // — same.
|
||||
}
|
||||
finally
|
||||
{
|
||||
PathfindRecorder.SetEnabled(false);
|
||||
if (File.Exists(path)) { File.Delete(path); }
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SetEnabled_TogglingTwice_IsIdempotent()
|
||||
{
|
||||
var path = NewTempPath();
|
||||
OverrideOutputPath(path);
|
||||
|
||||
try
|
||||
{
|
||||
PathfindRecorder.SetEnabled(true);
|
||||
Assert.True(PathfindRecorder.Enabled);
|
||||
|
||||
PathfindRecorder.SetEnabled(true); // second on: no-op, no error
|
||||
Assert.True(PathfindRecorder.Enabled);
|
||||
|
||||
PathfindRecorder.SetEnabled(false);
|
||||
Assert.False(PathfindRecorder.Enabled);
|
||||
|
||||
PathfindRecorder.SetEnabled(false); // second off: no-op
|
||||
Assert.False(PathfindRecorder.Enabled);
|
||||
}
|
||||
finally
|
||||
{
|
||||
PathfindRecorder.SetEnabled(false);
|
||||
if (File.Exists(path)) { File.Delete(path); }
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class RecorderStub : Server.Mobiles.BaseCreature
|
||||
{
|
||||
public RecorderStub(Serial serial) : base(serial)
|
||||
{
|
||||
Body = 0xC9;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -126,7 +126,7 @@ public class StepCacheFileTests
|
|||
}
|
||||
|
||||
[Fact]
|
||||
public void TryOpenLazyReader_TileDataHashMismatch_ReturnsFalse()
|
||||
public void TryOpenLazyReader_FingerprintMismatch_ReturnsFalse()
|
||||
{
|
||||
var cache = StepCache.Instance;
|
||||
cache.Clear();
|
||||
|
|
@ -139,7 +139,7 @@ public class StepCacheFileTests
|
|||
{
|
||||
cache.SaveToFile(path, map.MapID);
|
||||
|
||||
// Corrupt the TileDataHash field at byte offset 12 (Magic[4] + Version[4] + MapId[4]).
|
||||
// Corrupt the Fingerprint field at byte offset 12 (Magic[4] + Version[4] + MapId[4]).
|
||||
var bytes = File.ReadAllBytes(path);
|
||||
for (var i = 12; i < 20; i++)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -108,6 +108,8 @@ public class BitmapAStarAlgorithm : PathAlgorithm
|
|||
return null;
|
||||
}
|
||||
|
||||
Server.Engines.Pathing.PathfindRecorder.RecordIfEnabled(m, map, start, goal);
|
||||
|
||||
_currentMobileNeedsSlowPath = RequiresSlowPath(m);
|
||||
_currentMobilePlayerStrict = m.Player && m.AccessLevel < AccessLevel.GameMaster;
|
||||
if (m is BaseCreature creature)
|
||||
|
|
|
|||
|
|
@ -64,10 +64,21 @@ public sealed class StepCache
|
|||
/// counters are since-last-clear, not since-startup.
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
ClearResidentChunks();
|
||||
CloseLazyReaders();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drop all resident chunks AND zero counters, but keep lazy readers open.
|
||||
/// Useful in benchmark loops that want to measure "first query after boot" cost
|
||||
/// without paying the lazy-reader reopen overhead each iteration. Same intent as
|
||||
/// <see cref="Clear"/> minus the file-handle teardown.
|
||||
/// </summary>
|
||||
public void ClearResidentChunks()
|
||||
{
|
||||
_chunks.Clear();
|
||||
_keysList.Clear();
|
||||
CloseLazyReaders();
|
||||
_hits = 0;
|
||||
_missesNotBuilt = 0;
|
||||
_missesDirtyRebuild = 0;
|
||||
|
|
@ -83,6 +94,51 @@ public sealed class StepCache
|
|||
// stays bounded by MaxResidentChunks regardless of file size.
|
||||
private readonly Dictionary<int, StepCacheFile.LazyReader> _lazyReaders = new();
|
||||
|
||||
/// <summary>
|
||||
/// Combined XxHash3 fingerprint of the running server's TileData flag tables AND
|
||||
/// the per-map .mul / .uop file contents (mapX.mul, staidxX.mul, staticsX.mul).
|
||||
/// Public surface for tooling (benchmark fixtures, bake utilities) that wants to
|
||||
/// detect a stale .swb file without round-tripping through the lazy-open path.
|
||||
/// </summary>
|
||||
public static ulong ComputeLiveFingerprint(int mapId) => StepCacheFile.ComputeFingerprint(mapId);
|
||||
|
||||
/// <summary>
|
||||
/// Peek at a .swb file's stored fingerprint field without parsing the rest of the
|
||||
/// header. Returns false on missing file, bad magic, or wrong version.
|
||||
/// </summary>
|
||||
public static bool TryReadFingerprintFromFile(string path, out ulong fingerprint) =>
|
||||
StepCacheFile.TryReadFingerprint(path, out fingerprint);
|
||||
|
||||
/// <summary>
|
||||
/// Walk every chunk in <paramref name="mapId"/>, populate the resident set, then
|
||||
/// save to <paramref name="path"/>. Returns the number of chunks written.
|
||||
/// Designed for offline / fixture use; blocks the calling thread for many seconds
|
||||
/// on a full Trammel walk.
|
||||
/// </summary>
|
||||
public int BakeMap(int mapId, string path)
|
||||
{
|
||||
var map = Map.Maps[mapId];
|
||||
if (map == null || map == Map.Internal)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var chunkCols = (map.Width + ChunkSize - 1) / ChunkSize;
|
||||
var chunkRows = (map.Height + ChunkSize - 1) / ChunkSize;
|
||||
|
||||
for (var cy = 0; cy < chunkRows; cy++)
|
||||
{
|
||||
for (var cx = 0; cx < chunkCols; cx++)
|
||||
{
|
||||
// Any sourceZ works — the chunk is built on first access regardless of
|
||||
// whether the query returns Hit or Fallthrough_SourceZMismatch.
|
||||
TryGetMask(map, cx * ChunkSize, cy * ChunkSize, sourceZ: 0);
|
||||
}
|
||||
}
|
||||
|
||||
return SaveToFile(path, mapId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Persist all resident chunks for <paramref name="mapId"/> to a .swb file. Returns
|
||||
/// the number of chunks written. The file embeds a TileData fingerprint so a stale
|
||||
|
|
|
|||
|
|
@ -20,9 +20,12 @@ namespace Server.Engines.Pathing.Cache;
|
|||
/// u32 Magic = 0x42575300 ('SWB\0')
|
||||
/// u32 Version = current FormatVersion
|
||||
/// u32 MapId
|
||||
/// u64 TileDataHash XxHash3 over LandTable + ItemTable flags (via
|
||||
/// HashUtility); rejects a load when client tile data has
|
||||
/// shifted under us.
|
||||
/// u64 Fingerprint XxHash3 over (1) LandTable + ItemTable flags AND (2) the
|
||||
/// on-disk bytes of mapX.mul / .uop, staidxX.mul, staticsX.mul.
|
||||
/// Rejects a load when EITHER tile flags shifted (client patch)
|
||||
/// OR the map data was rewritten (CentredSharp / UOFiddler edit).
|
||||
/// The .mul format has no built-in CRC; this is the only way
|
||||
/// to detect those mutations.
|
||||
/// u64 BakeTimestamp DateTime.UtcNow.Ticks at write time (informational).
|
||||
/// u32 ChunkCount
|
||||
/// u64 IndexOffset File position where the chunk index begins.
|
||||
|
|
@ -54,7 +57,7 @@ internal static class StepCacheFile
|
|||
sizeof(uint) // Magic
|
||||
+ sizeof(uint) // Version
|
||||
+ sizeof(uint) // MapId
|
||||
+ sizeof(ulong) // TileDataHash
|
||||
+ sizeof(ulong) // Fingerprint
|
||||
+ sizeof(ulong) // BakeTimestamp
|
||||
+ sizeof(uint) // ChunkCount
|
||||
+ sizeof(ulong); // IndexOffset
|
||||
|
|
@ -73,27 +76,69 @@ internal static class StepCacheFile
|
|||
|
||||
/// <summary>
|
||||
/// Byte offset of the IndexOffset u64 within the header
|
||||
/// (Magic+Version+MapId+TileDataHash+BakeTimestamp+ChunkCount = 32). Patched after chunks land.
|
||||
/// (Magic+Version+MapId+Fingerprint+BakeTimestamp+ChunkCount = 32). Patched after chunks land.
|
||||
/// </summary>
|
||||
private const int IndexOffsetFieldPosition = 32;
|
||||
|
||||
public delegate bool ChunkEnumerator(out int chunkX, out int chunkY, out StepChunk chunk);
|
||||
|
||||
/// <summary>
|
||||
/// Computes a stable hash of the loaded TileData flags via XxHash3 (HashUtility).
|
||||
/// Bake files carry this hash so a load can refuse to populate the cache when tile
|
||||
/// data has shifted (client patch, mismatched version) — mismatched data would
|
||||
/// silently skew walkability answers. Hash is stable as long as HashUtility's seed
|
||||
/// constant doesn't change.
|
||||
/// Peek at a .swb file's Fingerprint field (header byte offset 12) without
|
||||
/// reading any chunk data. Returns false on missing file, bad magic, or wrong
|
||||
/// version. Cheap — reads 20 bytes total.
|
||||
/// </summary>
|
||||
public static ulong ComputeTileDataHash()
|
||||
public static bool TryReadFingerprint(string path, out ulong fingerprint)
|
||||
{
|
||||
fingerprint = 0;
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
|
||||
Span<byte> buf = stackalloc byte[20];
|
||||
if (stream.Read(buf) < 20)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (BinaryPrimitives.ReadUInt32LittleEndian(buf) != Magic)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (BinaryPrimitives.ReadUInt32LittleEndian(buf[4..]) != FormatVersion)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// mapId is at buf[8..12], we skip; hash is at buf[12..20].
|
||||
fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(buf[12..]);
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Combined XxHash3 fingerprint over (1) the loaded TileData flag tables and (2) the
|
||||
/// per-map .mul / .uop file contents (via <see cref="TileMatrix.MapFilesFingerprint"/>).
|
||||
/// Bake files carry this hash so a load can refuse to populate the cache when EITHER
|
||||
/// tile flags shifted (client patch) OR the map data was rewritten (CentredSharp /
|
||||
/// UOFiddler edit). The .mul format has no built-in CRC; this is the only way to
|
||||
/// detect those mutations.
|
||||
/// </summary>
|
||||
public static ulong ComputeFingerprint(int mapId)
|
||||
{
|
||||
var hasher = HashUtility.CreateXxHash3();
|
||||
|
||||
// TileData flag tables — same projection trick as before: just the Flags ulong
|
||||
// from each entry, written little-endian into a contiguous byte buffer. The
|
||||
// struct itself has a string Name (reference) whose object identity isn't
|
||||
// stable across runs, so MemoryMarshal.Cast over the whole struct would drift.
|
||||
var landTable = TileData.LandTable;
|
||||
var itemTable = TileData.ItemTable;
|
||||
|
||||
// Project just the Flags ulong from each entry into a contiguous byte buffer.
|
||||
// The struct itself contains a string Name (reference) whose object identity isn't
|
||||
// stable across runs, so we can't MemoryMarshal.Cast the whole struct.
|
||||
var bytes = new byte[(landTable.Length + itemTable.Length) * sizeof(ulong)];
|
||||
var span = bytes.AsSpan();
|
||||
|
||||
|
|
@ -106,8 +151,19 @@ internal static class StepCacheFile
|
|||
{
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(span[(itemOffset + i * 8)..], (ulong)itemTable[i].Flags);
|
||||
}
|
||||
hasher.Append(bytes);
|
||||
|
||||
return HashUtility.ComputeHash64(bytes);
|
||||
// Map files (mapX.mul / .uop, staidxX.mul, staticsX.mul). TileMatrix already
|
||||
// streamed them through XxHash3 once at construction; mix the result in.
|
||||
var map = Map.Maps[mapId];
|
||||
if (map != null && map != Map.Internal && map.Tiles != null)
|
||||
{
|
||||
Span<byte> mapHashBytes = stackalloc byte[sizeof(ulong)];
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(mapHashBytes, map.Tiles.MapFilesFingerprint);
|
||||
hasher.Append(mapHashBytes);
|
||||
}
|
||||
|
||||
return hasher.GetCurrentHashAsUInt64();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -128,7 +184,7 @@ internal static class StepCacheFile
|
|||
w.Write(Magic);
|
||||
w.Write(FormatVersion);
|
||||
w.Write(mapId);
|
||||
w.Write(ComputeTileDataHash());
|
||||
w.Write(ComputeFingerprint((int)mapId));
|
||||
w.Write((ulong)DateTime.UtcNow.Ticks);
|
||||
w.Write(chunkCount);
|
||||
w.Write(0UL); // IndexOffset placeholder, patched after chunks
|
||||
|
|
@ -172,7 +228,7 @@ internal static class StepCacheFile
|
|||
|
||||
/// <summary>
|
||||
/// Opens a .swb file and reads only its header + chunk-offset index. Returns null on
|
||||
/// missing file, magic / version mismatch, or TileDataHash mismatch (a stale bake
|
||||
/// missing file, magic / version mismatch, or Fingerprint mismatch (a stale bake
|
||||
/// against a freshly patched client). Callers own disposal of the returned reader.
|
||||
/// </summary>
|
||||
public static LazyReader OpenForLazy(string path)
|
||||
|
|
@ -213,12 +269,12 @@ internal static class StepCacheFile
|
|||
}
|
||||
|
||||
var mapId = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[8..]);
|
||||
var tileDataHash = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[12..]);
|
||||
var fingerprint = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[12..]);
|
||||
var bakeTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[20..]);
|
||||
var chunkCount = BinaryPrimitives.ReadUInt32LittleEndian(headerBuf[28..]);
|
||||
var indexOffset = BinaryPrimitives.ReadUInt64LittleEndian(headerBuf[32..]);
|
||||
|
||||
if (tileDataHash != ComputeTileDataHash())
|
||||
if (fingerprint != ComputeFingerprint((int)mapId))
|
||||
{
|
||||
stream.Dispose();
|
||||
return null;
|
||||
|
|
@ -243,7 +299,7 @@ internal static class StepCacheFile
|
|||
offsets[key] = off;
|
||||
}
|
||||
|
||||
return new LazyReader(stream, mapId, tileDataHash, bakeTimestamp, chunkCount, offsets);
|
||||
return new LazyReader(stream, mapId, fingerprint, bakeTimestamp, chunkCount, offsets);
|
||||
}
|
||||
catch
|
||||
{
|
||||
|
|
@ -352,7 +408,7 @@ internal static class StepCacheFile
|
|||
private byte[] _buffer;
|
||||
|
||||
public uint MapId { get; }
|
||||
public ulong TileDataHash { get; }
|
||||
public ulong Fingerprint { get; }
|
||||
public ulong BakeTimestamp { get; }
|
||||
public uint ChunkCount { get; }
|
||||
public int IndexedChunkCount => _offsets.Count;
|
||||
|
|
@ -360,13 +416,13 @@ internal static class StepCacheFile
|
|||
public bool Has(int chunkX, int chunkY) => _offsets.ContainsKey(PackChunkKey(chunkX, chunkY));
|
||||
|
||||
internal LazyReader(
|
||||
FileStream stream, uint mapId, ulong tileDataHash, ulong bakeTimestamp,
|
||||
FileStream stream, uint mapId, ulong fingerprint, ulong bakeTimestamp,
|
||||
uint chunkCount, Dictionary<ulong, ulong> offsets
|
||||
)
|
||||
{
|
||||
_stream = stream;
|
||||
MapId = mapId;
|
||||
TileDataHash = tileDataHash;
|
||||
Fingerprint = fingerprint;
|
||||
BakeTimestamp = bakeTimestamp;
|
||||
ChunkCount = chunkCount;
|
||||
_offsets = offsets;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ namespace Server.Engines.Pathing;
|
|||
/// [PathCacheClear — drop all cached chunks, close lazy readers, zero counters.
|
||||
/// [PathCacheSave — persist resident chunks per map to Data/Pathfinding/<mapId>.swb.
|
||||
/// [PathCacheLoad — open those files as lazy backing stores. Also runs at startup.
|
||||
/// [PathRecord — toggle JSONL telemetry capture for replay / benchmark corpora.
|
||||
/// </summary>
|
||||
public static class PathCacheCommands
|
||||
{
|
||||
|
|
@ -25,10 +26,13 @@ public static class PathCacheCommands
|
|||
8192
|
||||
);
|
||||
|
||||
PathfindRecorder.Configure();
|
||||
|
||||
CommandSystem.Register("PathCacheStats", AccessLevel.Administrator, OnPathCacheStats);
|
||||
CommandSystem.Register("PathCacheClear", AccessLevel.Administrator, OnPathCacheClear);
|
||||
CommandSystem.Register("PathCacheSave", AccessLevel.Administrator, OnPathCacheSave);
|
||||
CommandSystem.Register("PathCacheLoad", AccessLevel.Administrator, OnPathCacheLoad);
|
||||
CommandSystem.Register("PathRecord", AccessLevel.Administrator, OnPathRecord);
|
||||
AutoLoadAtStartup();
|
||||
}
|
||||
|
||||
|
|
@ -120,4 +124,43 @@ public static class PathCacheCommands
|
|||
$"StepCache: opened {openedMaps} map(s) for lazy loading (total readers open: {StepCache.Instance.OpenLazyReaderCount})."
|
||||
);
|
||||
}
|
||||
|
||||
[Usage("PathRecord [on|off|flush|status]")]
|
||||
[Description("Toggles PathfindRecorder. With no arg, reports state. 'on' enables JSONL capture of every Find call; 'off' disables and flushes; 'flush' forces a buffer flush without disabling.")]
|
||||
private static void OnPathRecord(CommandEventArgs e)
|
||||
{
|
||||
var arg = (e.Arguments.Length > 0 ? e.Arguments[0] : "status").ToLowerInvariant();
|
||||
var from = e.Mobile;
|
||||
switch (arg)
|
||||
{
|
||||
case "on":
|
||||
{
|
||||
PathfindRecorder.SetEnabled(true);
|
||||
from.SendMessage(PathfindRecorder.Enabled
|
||||
? $"PathRecord: ON, writing to {PathfindRecorder.OutputPath}"
|
||||
: "PathRecord: enable failed (see server log)");
|
||||
break;
|
||||
}
|
||||
case "off":
|
||||
{
|
||||
PathfindRecorder.SetEnabled(false);
|
||||
from.SendMessage("PathRecord: OFF");
|
||||
break;
|
||||
}
|
||||
case "flush":
|
||||
{
|
||||
PathfindRecorder.Flush();
|
||||
from.SendMessage($"PathRecord: flushed ({PathfindRecorder.RecordsWritten} records this session)");
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
from.SendMessage(
|
||||
$"PathRecord: {(PathfindRecorder.Enabled ? "ON" : "OFF")}, "
|
||||
+ $"path={PathfindRecorder.OutputPath}, records={PathfindRecorder.RecordsWritten}"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
170
Projects/UOContent/Engines/Pathing/PathfindRecorder.cs
Normal file
170
Projects/UOContent/Engines/Pathing/PathfindRecorder.cs
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
using System.IO;
|
||||
using System.Text;
|
||||
using Server.Logging;
|
||||
using Server.Mobiles;
|
||||
using Server.Text;
|
||||
|
||||
namespace Server.Engines.Pathing;
|
||||
|
||||
/// <summary>
|
||||
/// Admin-toggled telemetry: appends a JSONL line per pathfind request to a file.
|
||||
/// One record per BitmapAStarAlgorithm.Find call, capturing the inputs (start, goal,
|
||||
/// map, capability flags) needed to replay the scenario in benchmarks. Output format
|
||||
/// matches the corpus the BDN harness consumes.
|
||||
///
|
||||
/// Hot-toggleable at runtime via the [PathRecord admin command — no restart needed.
|
||||
/// <see cref="Configure"/> only seeds the initial state from server.cfg
|
||||
/// (pathfinding.recorder.enable, default false).
|
||||
///
|
||||
/// Holds a single StreamWriter open while recording; its internal buffer absorbs
|
||||
/// per-record writes without per-call File.Open / File.Append. Each record is built
|
||||
/// in a stack-allocated ValueStringBuilder (zero per-int allocation for the field
|
||||
/// formatting), then handed to the writer as a ReadOnlySpan<char>.
|
||||
///
|
||||
/// <b>Workload note:</b> intended for short bursts of capture (turn on, walk a region
|
||||
/// or trigger a scenario, turn off). On a busy server with hundreds of pathfinds
|
||||
/// per second, sustained recording can saturate the StreamWriter's 4 KB buffer and
|
||||
/// block the game thread on disk writes. A backpressure-aware async sink is a
|
||||
/// future enhancement if 24/7 capture becomes a use case.
|
||||
/// </summary>
|
||||
public static class PathfindRecorder
|
||||
{
|
||||
private static readonly ILogger logger = LogFactory.GetLogger(typeof(PathfindRecorder));
|
||||
|
||||
private static bool _enabled;
|
||||
private static string _outputPath;
|
||||
private static StreamWriter _writer;
|
||||
private static long _recordsWritten;
|
||||
|
||||
public static bool Enabled => _enabled;
|
||||
public static string OutputPath => _outputPath;
|
||||
public static long RecordsWritten => _recordsWritten;
|
||||
|
||||
public static void Configure()
|
||||
{
|
||||
_outputPath = ServerConfiguration.GetOrUpdateSetting(
|
||||
"pathfinding.recorder.path",
|
||||
Path.Combine(Core.BaseDirectory, "Data", "Pathfinding", "recordings", "pathfinds.jsonl")
|
||||
);
|
||||
|
||||
var startEnabled = ServerConfiguration.GetOrUpdateSetting("pathfinding.recorder.enable", false);
|
||||
if (startEnabled)
|
||||
{
|
||||
SetEnabled(true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Toggle recording. When enabling, opens an append-mode StreamWriter; when
|
||||
/// disabling, flushes + disposes it. Idempotent — calling twice with the same
|
||||
/// state is a no-op.
|
||||
/// </summary>
|
||||
public static void SetEnabled(bool enabled)
|
||||
{
|
||||
if (enabled == _enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (enabled)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(_outputPath) ?? ".");
|
||||
var stream = new FileStream(_outputPath, FileMode.Append, FileAccess.Write, FileShare.Read);
|
||||
_writer = new StreamWriter(stream, new UTF8Encoding(false));
|
||||
_enabled = true;
|
||||
logger.Information("PathfindRecorder enabled, writing to {Path}", _outputPath);
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
logger.Warning(ex, "PathfindRecorder: failed to open {Path} for write", _outputPath);
|
||||
_writer = null;
|
||||
_enabled = false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_enabled = false;
|
||||
try
|
||||
{
|
||||
_writer?.Flush();
|
||||
_writer?.Dispose();
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
logger.Warning(ex, "PathfindRecorder: error closing {Path}", _outputPath);
|
||||
}
|
||||
_writer = null;
|
||||
logger.Information("PathfindRecorder disabled ({Count} records this session)", _recordsWritten);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Force a flush of the writer's internal buffer to disk. Safe to call when
|
||||
/// disabled (no-op). Useful after a burst of recording when an admin wants to
|
||||
/// inspect the file without waiting for buffer fill or disable.
|
||||
/// </summary>
|
||||
public static void Flush()
|
||||
{
|
||||
try
|
||||
{
|
||||
_writer?.Flush();
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
logger.Warning(ex, "PathfindRecorder: flush failed for {Path}", _outputPath);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Capture one Find call. Hot path: cheap when disabled (single bool check).
|
||||
/// When enabled, formats one JSONL line and writes it through the StreamWriter's
|
||||
/// internal buffer — flush is amortized across many calls.
|
||||
/// </summary>
|
||||
public static void RecordIfEnabled(Mobile m, Map map, Point3D start, Point3D goal)
|
||||
{
|
||||
if (!_enabled || _writer == null || m == null || map == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var canSwim = false;
|
||||
var canFly = false;
|
||||
var canOpenDoors = false;
|
||||
var canMoveOverObstacles = false;
|
||||
if (m is BaseCreature bc)
|
||||
{
|
||||
canSwim = bc.CanSwim;
|
||||
canFly = bc.CanFly;
|
||||
canOpenDoors = bc.CanOpenDoors;
|
||||
canMoveOverObstacles = bc.CanMoveOverObstacles;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// One interpolation handles every numeric field with no per-int ToString
|
||||
// allocation; bool fields use explicit literal spans because JSON wants
|
||||
// lowercase "true"/"false" and bool.ToString() yields "True"/"False".
|
||||
using var vsb = ValueStringBuilder.Create(192);
|
||||
vsb.Append(
|
||||
$"{{\"Name\":\"recorded\",\"MapId\":{map.MapID},\"StartX\":{start.X},\"StartY\":{start.Y},\"StartZ\":{start.Z},\"GoalX\":{goal.X},\"GoalY\":{goal.Y},\"GoalZ\":{goal.Z},\"CanSwim\":"
|
||||
);
|
||||
vsb.Append(canSwim ? "true" : "false");
|
||||
vsb.Append(",\"CanFly\":");
|
||||
vsb.Append(canFly ? "true" : "false");
|
||||
vsb.Append(",\"CanOpenDoors\":");
|
||||
vsb.Append(canOpenDoors ? "true" : "false");
|
||||
vsb.Append(",\"CanMoveOverObstacles\":");
|
||||
vsb.Append(canMoveOverObstacles ? "true" : "false");
|
||||
vsb.Append("}\n");
|
||||
_writer.Write(vsb.AsSpan());
|
||||
_recordsWritten++;
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
logger.Warning(ex, "PathfindRecorder: write failed, disabling");
|
||||
SetEnabled(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -41,6 +41,7 @@
|
|||
<IncludeInPackage>false</IncludeInPackage>
|
||||
</ProjectReference>
|
||||
<PackageReference Include="LibDeflate.Bindings" Version="1.0.3" />
|
||||
<PackageReference Include="System.IO.Hashing" Version="10.0.6" />
|
||||
<PackageReference Include="MailKit" Version="4.16.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="10.0.6" />
|
||||
<PackageReference Include="CommunityToolkit.HighPerformance" Version="8.4.2" />
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue