From bae1144ee44197f9ae474bf9beeb368793a8e6e9 Mon Sep 17 00:00:00 2001 From: Fr0zka Date: Fri, 30 Oct 2026 09:04:04 +0100 Subject: [PATCH] research: idle workers at 30 m/s (L122, no production change) 98% of idle-with-holes windows wait on the expanded critical set; region compile runs 0.26 ms but its gate stays pending 18 ms p50. Broad early admission: near +15.4 pt but p99 13.4 ms (over the VR 11 ms frame); near-only: +9.9 pt but far holes +1.5 pt. Trace instrumentation kept as a patch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1 --- .../Linux122/AnalyzeL122Admission.py | 198 ++++++++++++ .../WorldAssemblerM3/Linux122/REPORT.md | 18 ++ .../Linux122/RunL122Admission.py | 111 +++++++ .../Linux122/RunL122Hashes.py | 117 +++++++ .../WorldAssemblerM3/Linux122/RunL122Pairs.py | 298 ++++++++++++++++++ .../Linux122/l122-trace.patch | 218 +++++++++++++ 6 files changed, 960 insertions(+) create mode 100644 Standalone/Research/WorldAssemblerM3/Linux122/AnalyzeL122Admission.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux122/REPORT.md create mode 100644 Standalone/Research/WorldAssemblerM3/Linux122/RunL122Admission.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux122/RunL122Hashes.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux122/RunL122Pairs.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux122/l122-trace.patch diff --git a/Standalone/Research/WorldAssemblerM3/Linux122/AnalyzeL122Admission.py b/Standalone/Research/WorldAssemblerM3/Linux122/AnalyzeL122Admission.py new file mode 100644 index 0000000..58668a4 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux122/AnalyzeL122Admission.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Summarize L122 scheduler blockers and Core region compile timing.""" +from __future__ import annotations + +import json +import math +import statistics +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +TRACE_DIR = ROOT / "Saved/Linux/l122/diagnostic/unreal" +OUT = ROOT / "Saved/Linux/l122/diagnostic/analysis.json" + + +def read_events(path: Path) -> list[dict]: + rows = [] + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith("{"): + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1)) + return ordered[index] + + +def distribution(values: list[float]) -> dict: + if not values: + return {"n": 0, "p50": None, "p95": None, "max": None} + return { + "n": len(values), "p50": statistics.median(values), + "p95": percentile(values, 0.95), "max": max(values), + } + + +def summarize_run(path: Path) -> tuple[dict, list[dict], list[dict]]: + events = read_events(path) + stop = next(float(event["t_s"]) for event in events + if event.get("type") == "streaming_test_move_stopped") + move_start = min(float(event["t_s"]) for event in events + if event.get("type") == "streaming_sample" + and float(event.get("speed_cm_s", 0.0)) >= 50.0) + scheduler = [event for event in events if event.get("type") == "scheduler_admission" + and move_start <= float(event.get("t_s", 0.0)) < stop + and event.get("moving", 0)] + idle = [event for event in scheduler + if event.get("running_workers", 0) == 0 + and sum(event.get("desired_missing_by_lod", [0] * 9)[:3]) + + sum(event.get("pending_by_lod", [0] * 9)[:3]) > 0] + samples = [event for event in events if event.get("type") == "streaming_sample" + and float(event.get("speed_cm_s", 0.0)) >= 50.0] + near = [] + far_holes = [] + for event in samples: + desired = event.get("desired_by_lod", [0] * 9) + resident = event.get("resident_by_lod", [0] * 9) + covered = event.get("covered_by_lod", resident) + near.append(100.0 * sum(resident[:3]) / max(1, sum(desired[:3]))) + far_holes.append(100.0 * (sum(desired[3:]) - sum(covered[3:])) + / max(1, sum(desired[3:]))) + ticks = [float(event.get("tick_ms", 0.0)) for event in events + if event.get("type") == "game_thread_work_frame" + and move_start <= float(event.get("t_s", 0.0)) < stop] + + def near_array(event: dict, name: str) -> list[int]: + return event.get(name, [0] * 9)[:3] + + def near_count(event: dict, name: str) -> int: + return sum(near_array(event, name)) + + reason_counts = { + "core_region_pending": sum(event.get("core_region_compile_pending", 0) > 0 + for event in idle), + "core_region_exact_near_decline": sum( + sum(near_array(event, "region_compile_declines_by_lod")) > 0 for event in idle), + "far_parent_near_decline": sum( + sum(near_array(event, "far_parent_declines_by_lod")) > 0 for event in idle), + "near_cap_decline": sum(any(value in (1, 2) + for value in event.get("declined_rule_by_lod", [0] * 9)[:3]) + for event in idle), + "critical_set_not_ready": sum(event.get("critical_tiles_ready", 1) == 0 + for event in idle), + "walkable_ring_not_ready": sum(event.get("walkable_ready", 1) == 0 for event in idle), + "pawn_collision_gate_held": sum(event.get("pawn_collision_gate_engaged", 0) > 0 + for event in idle), + "desired_center_stale": sum(event.get("desired_center_current", 1) == 0 for event in idle), + "only_pending_near_work": sum(near_count(event, "desired_missing_by_lod") == 0 + and near_count(event, "pending_by_lod") > 0 for event in idle), + "no_near_admission_this_frame": sum( + near_count(event, "submitted_by_lod") == 0 for event in idle), + } + + compiles = [event for event in events if event.get("type") == "core_region_compile_complete"] + requests = {event.get("request_serial"): event for event in events + if event.get("type") == "core_region_compile_request"} + starts = [event for event in events if event.get("type") == "core_region_compile_start"] + request_start = [ + (float(event["t_s"]) - float(requests[event["request_serial"]]["t_s"])) * 1000.0 + for event in starts if event.get("request_serial") in requests + ] + near_missing = [near_count(event, "desired_missing_by_lod") for event in idle] + near_pending = [near_count(event, "pending_by_lod") for event in idle] + occupancy = [float(event.get("admission_occupancy", 0)) for event in idle] + row = { + "trace": str(path.relative_to(ROOT)), + "moving_scheduler_samples": len(scheduler), + "idle_missing_near_windows": len(idle), + "idle_window_share_pct": 100.0 * len(idle) / max(1, len(scheduler)), + "near_ready_moving_mean_pct": statistics.mean(near) if near else None, + "far_holes_moving_mean_pct": statistics.mean(far_holes) if far_holes else None, + "tick_p99_ms": percentile(ticks, 0.99), "tick_max_ms": max(ticks, default=None), + "idle_near_unadmitted_mean": statistics.mean(near_missing) if near_missing else None, + "idle_near_pending_mean": statistics.mean(near_pending) if near_pending else None, + "idle_admission_occupancy_p95": percentile(occupancy, 0.95), + "idle_admission_occupancy_max": max(occupancy, default=None), + "idle_critical_missing_p50": statistics.median( + [float(event.get("critical_missing", 0)) for event in idle]) if idle else None, + "idle_reasons": reason_counts, + "region_compile_request_start_ms": distribution(request_start), + "region_compile_task_queue_ms": distribution( + [float(event.get("compile_queue_ms", 0)) for event in compiles]), + "region_compile_task_runtime_ms": distribution( + [float(event.get("compile_task_ms", 0)) for event in compiles]), + "region_compile_core_wall_ms": distribution( + [float(event.get("compile_core_wall_ms", 0)) for event in compiles]), + "region_compile_pending_gate_ms": distribution( + [float(event.get("pending_gate_elapsed_ms", 0)) for event in compiles]), + "region_compiled_tile_batches": sum(event.get("compiled_tiles", 0) > 0 + for event in compiles), + "region_compile_worker_thread_count": len({event.get("compile_worker_thread_id") + for event in compiles}), + "region_compile_submitted_while_generation_idle": sum( + event.get("active_generation_workers", -1) == 0 for event in starts), + "region_compile_requests": len(requests), + "region_compile_starts": len(starts), + "region_compile_completions": len(compiles), + } + return row, idle, compiles + + +def main() -> int: + traces = sorted(TRACE_DIR.glob("fast-r*.trace.jsonl")) + if len(traces) != 5: + raise SystemExit(f"expected five diagnostic traces, found {len(traces)} in {TRACE_DIR}") + runs = [summarize_run(path) for path in traces] + rows = [run[0] for run in runs] + idles = [event for run in runs for event in run[1]] + compiles = [event for run in runs for event in run[2]] + total_scheduler = sum(row["moving_scheduler_samples"] for row in rows) + reason_names = rows[0]["idle_reasons"].keys() + summary = { + "runs": rows, + "pooled_idle_windows": len(idles), + "pooled_scheduler_samples": total_scheduler, + "pooled_idle_window_share_pct": 100.0 * len(idles) / max(1, total_scheduler), + "pooled_idle_reasons": { + reason: sum(row["idle_reasons"][reason] for row in rows) + for reason in reason_names + }, + "pooled_near_ready_moving_mean_pct": statistics.mean( + row["near_ready_moving_mean_pct"] for row in rows), + "pooled_tick_p99_ms": distribution( + [float(row["tick_p99_ms"]) for row in rows if row["tick_p99_ms"] is not None]), + "pooled_tick_max_ms": max(float(row["tick_max_ms"]) for row in rows + if row["tick_max_ms"] is not None), + "pooled_region_compile_task_queue_ms": distribution( + [float(event.get("compile_queue_ms", 0)) for event in compiles]), + "pooled_region_compile_task_runtime_ms": distribution( + [float(event.get("compile_task_ms", 0)) for event in compiles]), + "pooled_region_compile_core_wall_ms": distribution( + [float(event.get("compile_core_wall_ms", 0)) for event in compiles]), + "pooled_region_compile_pending_gate_ms": distribution( + [float(event.get("pending_gate_elapsed_ms", 0)) for event in compiles]), + "pooled_region_compiled_tile_batches": sum( + event.get("compiled_tiles", 0) > 0 for event in compiles), + "pooled_region_compile_worker_thread_count": len({ + event.get("compile_worker_thread_id") for event in compiles}), + "pooled_region_compile_submitted_while_generation_idle": sum( + event.get("active_generation_workers", -1) == 0 + for run in runs for event in read_events(ROOT / run[0]["trace"]) + if event.get("type") == "core_region_compile_start"), + "pooled_region_compile_requests": sum(row["region_compile_requests"] for row in rows), + } + OUT.parent.mkdir(parents=True, exist_ok=True) + OUT.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux122/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux122/REPORT.md new file mode 100644 index 0000000..b0c5014 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux122/REPORT.md @@ -0,0 +1,18 @@ +# Linux 122 — idle workers at speed + +**Decision:** keep the L121 admission policy. Neither tested fast-travel relaxation met every coverage and tick limit. + +Five diagnostic runs at 30 m/s had 686 idle-worker windows with near tiles missing. In 673 (98%) the expanded critical set was not ready; 462 (67%) also had Core-region compilation pending, and 441 (64%) declined an exact region-dependent near tile. These counts overlap. Far-parent, near-cap, walkable-ring, pawn-collision, and stale-center holds were all zero. Only 13 windows had pending near work and no unadmitted near tile. The desired center was current in every window. + +Region compile requests scheduled immediately (request to task submission p50/p95 **0.008/0.021 ms**). UE task queue delay was **0.122/14.856 ms** p50/p95 (max 54.4 ms); task runtime was **0.485/4.572 ms**, with Core compile wall time **0.259/3.679 ms**. The task runs on UE background workers, not the Game Thread. It was submitted while generation workers were idle in 680/1,158 requests. The compile targets the current desired center; there is no velocity look-ahead. The completion gate remained pending **18.216/21.136 ms** p50/p95 while the Game Thread observed completion. + +The UE/headless coverage gap is not a denominator mismatch: both use resident near tiles / desired LOD0–2 tiles, with about 1,346–1,348 desired near tiles in UE versus 1,344 in the saved headless runs. Headless was about 82.6%; UE was 45.0%. The measured idle windows point to UE admission. Apply work is small (L121 near apply and proxy p95 were about 0.20 and 0.24 ms), though worker-finish-to-apply wait reached 120 ms p95 and may add secondary delay. + +| Fast-travel design | Near change | Far-hole change | Paired p99 change | Worst tick | +|---|---:|---:|---:|---:| +| Admit normal near/far shares before expanded critical readiness | +15.42 pp | −0.28 pp | +0.28 ms | 14.52 ms | +| Admit near LOD0–2 only until critical readiness | +9.89 pp | +1.48 pp | −0.56 ms | 14.38 ms | + +The first design exceeded the p99 bar (worst per-run p99 13.36 ms versus L121's 11.00 ms); the second increased far holes. Both stayed under L121's 14.74 ms worst-tick ceiling. No admission behavior was retained, so arrival behavior remains L121. + +UE build passed with `-NoUBA -MaxParallelActions=4`. Exact tile and whole-view hashes matched L104 in **24/24** Clang/GCC W1/W8/W12 runs. Fast tier passed (**269 gates, 5 diagnostics**). Logs are under `Saved/Linux/l122`. diff --git a/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Admission.py b/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Admission.py new file mode 100644 index 0000000..dbc5140 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Admission.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Capture five 30 m/s UE runs with admission-reason tracing enabled.""" +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +HOST = Path("/home/jahni/VFHost") +EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd") +OUT = ROOT / "Saved/Linux/l122/diagnostic/unreal" +HOST_OUT = HOST / "Plugins/VoxelForge/Saved/Linux/l122/diagnostic" +HOST_SAVED = HOST / "Saved/Linux/l122" +SPAWN_M = (19.166015625, -14.2001953125, -753.3740234375) + + +def run(repetition: int) -> dict: + trace = OUT / f"fast-r{repetition}.trace.jsonl" + abslog = OUT / f"fast-r{repetition}.log" + stdout = OUT / f"fast-r{repetition}.stdout.log" + export = HOST_OUT / f"fast-r{repetition}" + userdir = HOST_SAVED / "user" / f"fast-r{repetition}" + zendata = HOST_SAVED / "zen" / f"fast-r{repetition}" + ddc = HOST_SAVED / "DDC" + for path in (export, userdir, zendata, ddc): + path.mkdir(parents=True, exist_ok=True) + spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M) + command = [ + str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", + "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", + "-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer", + "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", + f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", + "-archetype=TunnelNetwork", "-slot=0", "-modes=export", "-gameconfig=1", + "-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25", + "-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0", + "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0", + f"-out={export}", "-voxel.StreamingCoarseFirst=1", + "-voxel.StreamingDynamicNearReserve=1", + "-voxel.StreamingLegacyPendingResultCap=0", + "-voxel.CoreRegionConcurrentAdmission=1", "-voxel.StreamingResultBacklogCap=256", + "-voxel.StreamingReadyQueueDepth=8", "-voxel.StreamingBackgroundWorkerCap=1", + "-voxel.CoreColumnCacheDirectMapped=0", "-voxel.StreamingFarApplySupplement=1", + "-voxel.CoreMCGradientMemoization=1", "-voxel.StreamingCollisionLookahead=1", + "-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=3000.0", + "-voxel.TestMoveStartSeconds=6.0", "-voxel.TestMoveDurationSeconds=12.0", + "-voxel.TestExitSeconds=23.0", "-voxel.TestMoveDirection=1,0,0", + "-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0", + "-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}", + "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", + "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", + f"-abslog={abslog}", + ] + OUT.mkdir(parents=True, exist_ok=True) + wrapped = ["taskset", "-c", "0-11", "timeout", "1740s", *command] + started = time.monotonic() + with stdout.open("w", encoding="utf-8") as stream: + result = subprocess.run(wrapped, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + events = [] + if trace.is_file(): + for line in trace.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith("{"): + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + continue + moving = [event for event in events if event.get("type") == "streaming_sample" + and float(event.get("speed_cm_s", 0.0)) >= 50.0] + dwell = [event for event in events if event.get("type") == "streaming_sample" + and float(event.get("speed_cm_s", 0.0)) < 50.0 + and float(event.get("t_s", 0.0)) >= 18.0] + backend = next((event for event in events if event.get("type") == "worker_backend_config"), {}) + cap = next((event for event in events if event.get("type") == "streaming_worker_cap_readback"), {}) + if result.returncode or len(moving) < 40 or len(dwell) < 16: + raise RuntimeError(f"incomplete Unreal run {repetition}: exit={result.returncode}, " + f"move={len(moving)}, dwell={len(dwell)}, trace={trace}") + if (int(backend.get("streaming_far_apply_supplement", -1)) != 1 + or int(cap.get("streaming_background_worker_cap", -1)) != 1 + or int(backend.get("effective_tile_slots", -1)) != 8): + raise RuntimeError(f"configuration readback mismatch: {backend}; {cap}") + if not any(event.get("type") == "scheduler_admission" + and "region_compile_declines_by_lod" in event + for event in events): + raise RuntimeError(f"admission diagnostics missing from {trace}") + row = { + "platform": "unreal", "phase": "diagnostic_baseline", "repetition": repetition, + "exit_code": result.returncode, "wall_s": round(time.monotonic() - started, 3), + "moving_samples": len(moving), "dwell_samples": len(dwell), + "trace": str(trace.relative_to(ROOT)), "log": str(abslog.relative_to(ROOT)), + } + with (OUT / "runs.jsonl").open("a", encoding="utf-8") as stream: + stream.write(json.dumps(row, sort_keys=True) + "\n") + print("[L122Admission] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + if not (ROOT / "Standalone/data/owner_world.json").is_file() or not EDITOR.is_file(): + raise SystemExit("missing owner world data or Unreal editor") + OUT.mkdir(parents=True, exist_ok=True) + runs = [run(repetition) for repetition in range(1, 6)] + (OUT / "summary.json").write_text(json.dumps({"runs": runs}, indent=2) + "\n", + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Hashes.py b/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Hashes.py new file mode 100644 index 0000000..c23bbe5 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Hashes.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Compare L122 exact tile hashes with L104 across all strata, compilers and workers.""" +from __future__ import annotations + +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l122/hashes" +DATA = ROOT / "Standalone/data/owner_world.json" +OLD = ROOT / "Saved/Linux/l104/hashes" +BINARIES = { + "clang": ROOT / "Saved/Linux/l122/verify/bin/VoxelForgeCoreHeadless-verify-clang", + "gcc": ROOT / "Saved/Linux/l122/verify/bin/VoxelForgeCoreHeadless-verify-gcc", +} +CAMERAS = { + "sw": ((8.0, 16.0, -372.2626953125), 2850), + "fi": ((25.8369140625, 3.19921875, -737.0), 2249), + "tn": ((19.166015625, -14.2001953125, -753.3740234375), 2233), + "vs": ((-20.0625, -5.068359375, -900.353515625), 2329), +} +TILE = re.compile( + r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) " + r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) " + r"topology=(0x[0-9a-f]+)$", re.M) +WHOLE = re.compile( + r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) " + r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M) + + +def parse(path: Path) -> tuple[dict, tuple[str, ...], str]: + content = path.read_text(encoding="utf-8", errors="replace") + tiles = {(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))): + m.group(5, 6, 7, 8) for m in TILE.finditer(content)} + whole, stamp = WHOLE.search(content), STAMP.search(content) + if whole is None or stamp is None or len(tiles) != int(whole.group(1)): + raise RuntimeError(f"incomplete hash output: {path}") + return tiles, whole.groups(), stamp.group(1) + + +def run(stratum: str, compiler: str, workers: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + log = OUT / f"{stratum}-{compiler}-w{workers}.log" + camera, expected_tiles = CAMERAS[stratum] + camera_arg = ",".join(f"{v:.12f}" for v in camera) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARIES[compiler]), str(DATA), "--world-assembler-prototype", + "--stream-benchmark", "--stream-pipeline", f"--workers={workers}", + "--clip-radius=3", "--max-clip-level=8", "--max-level=8", + f"--camera={camera_arg}", "--stream-hash-tiles", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + process = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + if process.returncode: + raise RuntimeError(f"hash process exited {process.returncode}: {log}") + tiles, whole, stamp = parse(log) + if len(tiles) != expected_tiles: + raise RuntimeError(f"expected {expected_tiles} tiles, got {len(tiles)}: {log}") + old_log = OLD / f"{stratum}-{compiler}-w{workers}.log" + old_tiles, old_whole, _old_stamp = parse(old_log) + clang_reference, clang_whole, _ = parse(OLD / f"{stratum}-clang-w1.log") + + def difference(reference: dict, candidate: dict) -> tuple[int, int]: + keys = set(reference) | set(candidate) + mismatches = sum(reference.get(key) != candidate.get(key) for key in keys) + return mismatches, len(keys) + + old_mismatches, old_keys = difference(old_tiles, tiles) + cross_mismatches, cross_keys = difference(clang_reference, tiles) + row = { + "stratum": stratum, "compiler": compiler, "workers": workers, + "source_stamp": stamp, "tiles": len(tiles), + "l104_same_cell_tile_mismatches": old_mismatches, + "l104_same_cell_key_count": old_keys, + "l104_same_cell_whole_hash_match": whole == old_whole, + "clang_w1_tile_mismatches": cross_mismatches, + "clang_w1_key_count": cross_keys, + "clang_w1_whole_hash_match": whole == clang_whole, + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT)), + } + print("[L122HashRun] " + json.dumps(row, sort_keys=True), flush=True) + row["tile_hashes"] = {"|".join(map(str, key)): value for key, value in tiles.items()} + return row + + +def main() -> int: + for path in (*BINARIES.values(), DATA): + if not path.is_file(): + raise SystemExit(f"missing input: {path}") + rows = [] + for stratum in CAMERAS: + for compiler in BINARIES: + for workers in (1, 8, 12): + rows.append(run(stratum, compiler, workers)) + (OUT / "runs-partial.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + failures = [row for row in rows if row["l104_same_cell_tile_mismatches"] + or not row["l104_same_cell_whole_hash_match"] + or row["clang_w1_tile_mismatches"] + or not row["clang_w1_whole_hash_match"]] + summary = {"runs": [{key: value for key, value in row.items() if key != "tile_hashes"} + for row in rows], "failed_runs": len(failures)} + (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", + encoding="utf-8") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Pairs.py b/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Pairs.py new file mode 100644 index 0000000..ed04482 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux122/RunL122Pairs.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Archived five-pair harness for transient L122 fast-travel admission candidates. + +The candidate CVar branches were removed after neither design met the quality gate. +""" +from __future__ import annotations + +import argparse +import json +import math +import statistics +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +HOST = Path("/home/jahni/VFHost") +EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd") +SPAWN_TN_M = (19.166015625, -14.2001953125, -753.3740234375) +SPAWN_FI_M = (25.8369140625, 3.19921875, -737.0) +DESIGN = "relax_expanded_critical_gate" +NEAR_ONLY_DESIGN = "near_only_expanded_critical_gate" + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + return ordered[max(0, min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1))] + + +def read_events(path: Path) -> list[dict]: + rows = [] + if path.is_file(): + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not line.startswith("{"): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def run(scenario: str, condition: str, repetition: int, design: str) -> dict: + candidate = condition == design + wait_for_critical = 0 if candidate else 1 + near_only_while_critical_pending = int(candidate and design == NEAR_ONLY_DESIGN) + if scenario == "movement": + speed_cm_s, start_s, duration_s, exit_s, slot = 3000.0, 6.0, 12.0, 23.0, 0 + spawn = SPAWN_TN_M + else: + speed_cm_s, start_s, duration_s, exit_s, slot = 140.0, 20.0, 12.0, 37.0, 1 + spawn = SPAWN_FI_M + + output_name = scenario + ("-near-only" if design == NEAR_ONLY_DESIGN else "") + root_out = ROOT / f"Saved/Linux/l122/{output_name}" + out = root_out / condition / "unreal" + host_out = HOST / "Plugins/VoxelForge/Saved/Linux/l122" / output_name / condition + host_saved = HOST / "Saved/Linux/l122" + label = f"{scenario}-r{repetition}" + trace = out / f"{label}.trace.jsonl" + abslog = out / f"{label}.log" + stdout = out / f"{label}.stdout.log" + export = host_out / "output" / label + userdir = host_saved / "user" / scenario / condition / label + zendata = host_saved / "zen" / scenario / condition / label + ddc = host_saved / "DDC" + for directory in (out, export, userdir, zendata, ddc): + directory.mkdir(parents=True, exist_ok=True) + trace.unlink(missing_ok=True) + spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in spawn) + command = [ + str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", + "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", + "-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer", + "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", + f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", + "-archetype=TunnelNetwork", f"-slot={slot}", "-modes=export", "-gameconfig=1", + "-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25", + "-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0", + "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0", + f"-out={export}", "-voxel.StreamingCoarseFirst=1", + "-voxel.StreamingDynamicNearReserve=1", + "-voxel.StreamingLegacyPendingResultCap=0", + "-voxel.CoreRegionConcurrentAdmission=1", "-voxel.StreamingResultBacklogCap=256", + "-voxel.StreamingReadyQueueDepth=8", "-voxel.StreamingBackgroundWorkerCap=1", + "-voxel.CoreColumnCacheDirectMapped=0", "-voxel.StreamingFarApplySupplement=1", + f"-voxel.StreamingFastTravelWaitForAllCriticalTiles={wait_for_critical}", + f"-voxel.StreamingFastTravelNearOnlyWhileCriticalPending={near_only_while_critical_pending}", + "-voxel.CoreMCGradientMemoization=1", "-voxel.StreamingCollisionLookahead=1", + "-voxel.TestMove=1", f"-voxel.TestMoveSpeedCmPerSecond={speed_cm_s:.1f}", + f"-voxel.TestMoveStartSeconds={start_s:.1f}", + f"-voxel.TestMoveDurationSeconds={duration_s:.1f}", + f"-voxel.TestExitSeconds={exit_s:.1f}", "-voxel.TestMoveDirection=1,0,0", + "-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0", + "-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}", + "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", + "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", + f"-abslog={abslog}", + ] + wrapped = ["taskset", "-c", "0-11", "timeout", "1740s", *command] + started = time.monotonic() + with stdout.open("w", encoding="utf-8") as stream: + process = subprocess.run(wrapped, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + events = read_events(trace) + samples = [event for event in events if event.get("type") == "streaming_sample"] + moving = [event for event in samples if float(event.get("speed_cm_s", 0.0)) >= 50.0] + stop = next((event for event in events if event.get("type") == "streaming_test_move_stopped"), {}) + stop_s = float(stop.get("t_s", 0.0)) + move_start = min((float(event.get("t_s", 0.0)) for event in moving), default=float("inf")) + dwell = [event for event in samples if float(event.get("t_s", 0.0)) >= stop_s + and float(event.get("speed_cm_s", 0.0)) < 50.0] + backend = next((event for event in events if event.get("type") == "worker_backend_config"), {}) + cap = next((event for event in events if event.get("type") == "streaming_worker_cap_readback"), {}) + if process.returncode or len(moving) < 40 or len(dwell) < 16: + raise RuntimeError(f"incomplete {scenario} run: {trace}; exit={process.returncode}; " + f"move={len(moving)} dwell={len(dwell)}") + if (int(backend.get("streaming_fast_travel_wait_for_all_critical_tiles", -1)) + != wait_for_critical + or int(backend.get("streaming_fast_travel_near_only_while_critical_pending", -1)) + != near_only_while_critical_pending + or int(backend.get("streaming_far_apply_supplement", -1)) != 1 + or int(cap.get("streaming_background_worker_cap", -1)) != 1 + or int(backend.get("effective_tile_slots", -1)) != 8): + raise RuntimeError(f"configuration readback mismatch: {backend}; {cap}") + + moving_scheduler = [event for event in events if event.get("type") == "scheduler_admission" + and move_start <= float(event.get("t_s", 0.0)) < stop_s + and event.get("moving", 0)] + idle = [event for event in moving_scheduler + if event.get("running_workers", 0) == 0 + and sum(event.get("desired_missing_by_lod", [0] * 9)[:3]) + + sum(event.get("pending_by_lod", [0] * 9)[:3]) > 0] + move_rows = [] + for event in moving: + desired = event.get("desired_by_lod", [0] * 9) + resident = event.get("resident_by_lod", [0] * 9) + covered = event.get("covered_by_lod", resident) + move_rows.append({ + "near_ready_pct": 100.0 * sum(resident[:3]) / max(1, sum(desired[:3])), + "lod0_ready_pct": 100.0 * resident[0] / max(1, desired[0]), + "lod1_ready_pct": 100.0 * resident[1] / max(1, desired[1]), + "lod2_ready_pct": 100.0 * resident[2] / max(1, desired[2]), + "far_hole_pct": 100.0 * (sum(desired[3:]) - sum(covered[3:])) + / max(1, sum(desired[3:])), + }) + work_frames = [event for event in events if event.get("type") == "game_thread_work_frame" + and move_start <= float(event.get("t_s", 0.0)) < stop_s] + tick_values = [float(event.get("tick_ms", 0.0)) for event in work_frames] + update_values = [float(event.get("update_chunks_ms", 0.0)) for event in work_frames] + requests = [event for event in events if event.get("type") == "request" + and move_start <= float(event.get("t_s", 0.0)) < stop_s + and event.get("level", 9) <= 2] + + arrival_rows = [event for event in samples if float(event.get("t_s", 0.0)) < start_s + and float(event.get("speed_cm_s", 0.0)) < 50.0] + + def near_complete(event: dict) -> bool: + desired = event.get("desired_by_lod", [0] * 9) + settled = event.get("settled_by_lod", [0] * 9) + return sum(desired[:3]) > 0 and sum(settled[:3]) >= sum(desired[:3]) + + def whole_complete(event: dict) -> bool: + desired = event.get("desired_by_lod", [0] * 9) + settled = event.get("settled_by_lod", [0] * 9) + return sum(desired) > 0 and sum(settled) >= sum(desired) + + def far_holes(event: dict) -> float | None: + desired = event.get("desired_by_lod", [0] * 9)[3:] + covered = event.get("covered_by_lod", [0] * 9)[3:] + return (100.0 * (sum(desired) - sum(covered)) / sum(desired) + if sum(desired) else None) + + near_settled = next((event for event in arrival_rows if near_complete(event)), None) + whole_settled = next((event for event in arrival_rows if whole_complete(event)), None) + far_clear = next((event for event in arrival_rows if far_holes(event) == 0.0), None) + + def mean_metric(name: str, rows: list[dict]) -> float | None: + values = [float(row[name]) for row in rows if row.get(name) is not None] + return statistics.mean(values) if values else None + + row = { + "scenario": scenario, "condition": condition, "repetition": repetition, + "design": design, "speed_mps": speed_cm_s / 100.0, + "wait_for_all_critical_tiles": wait_for_critical, + "near_only_while_critical_pending": near_only_while_critical_pending, + "exit_code": process.returncode, "wall_s": round(time.monotonic() - started, 3), + "moving_samples": len(moving), "dwell_samples": len(dwell), + "near_ready_moving_mean_pct": mean_metric("near_ready_pct", move_rows), + "lod0_ready_moving_mean_pct": mean_metric("lod0_ready_pct", move_rows), + "lod1_ready_moving_mean_pct": mean_metric("lod1_ready_pct", move_rows), + "lod2_ready_moving_mean_pct": mean_metric("lod2_ready_pct", move_rows), + "far_holes_moving_mean_pct": mean_metric("far_hole_pct", move_rows), + "game_thread_tick_p99_ms": percentile(tick_values, 0.99), + "game_thread_tick_max_ms": max(tick_values, default=None), + "stream_update_p99_ms": percentile(update_values, 0.99), + "stream_update_max_ms": max(update_values, default=None), + "near_requests_during_move": len(requests), + "arrival_sample_count": len(arrival_rows), + "lod02_settled_t_s": float(near_settled["t_s"]) if near_settled else None, + "whole_view_settled_t_s": float(whole_settled["t_s"]) if whole_settled else None, + "far_clear_t_s": float(far_clear["t_s"]) if far_clear else None, + "far_holes_at_pre_move_pct": far_holes(arrival_rows[-1]) if arrival_rows else None, + "idle_missing_near_windows": len(idle), + "idle_critical_gate_closed": sum(not event.get("critical_tiles_ready", 1) + for event in idle), + "idle_exact_region_declines": sum( + sum(event.get("region_compile_declines_by_lod", [0] * 9)[:3]) > 0 + for event in idle), + "idle_far_parent_declines": sum( + sum(event.get("far_parent_declines_by_lod", [0] * 9)[:3]) > 0 + for event in idle), + "idle_near_cap_declines": sum( + any(value in (1, 2) for value in event.get("declined_rule_by_lod", [0] * 9)[:3]) + for event in idle), + "idle_desired_center_stale": sum(not event.get("desired_center_current", 1) + for event in idle), + "trace": str(trace.relative_to(ROOT)), "log": str(abslog.relative_to(ROOT)), + } + print("[L122Pair] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def pairs_for(runs: list[dict], scenario: str, design: str) -> list[dict]: + rows = [] + for repetition in range(1, 6): + baseline = next(row for row in runs if row["scenario"] == scenario + and row["condition"] == "baseline" and row["repetition"] == repetition) + candidate = next(row for row in runs if row["scenario"] == scenario + and row["condition"] == design and row["repetition"] == repetition) + rows.append({ + "repetition": repetition, + "near_ready_delta_pct": candidate.get("near_ready_moving_mean_pct", 0.0) + - baseline.get("near_ready_moving_mean_pct", 0.0), + "lod0_ready_delta_pct": candidate.get("lod0_ready_moving_mean_pct", 0.0) + - baseline.get("lod0_ready_moving_mean_pct", 0.0), + "lod1_ready_delta_pct": candidate.get("lod1_ready_moving_mean_pct", 0.0) + - baseline.get("lod1_ready_moving_mean_pct", 0.0), + "lod2_ready_delta_pct": candidate.get("lod2_ready_moving_mean_pct", 0.0) + - baseline.get("lod2_ready_moving_mean_pct", 0.0), + "far_holes_delta_pct": candidate.get("far_holes_moving_mean_pct", 0.0) + - baseline.get("far_holes_moving_mean_pct", 0.0), + "tick_p99_delta_ms": candidate.get("game_thread_tick_p99_ms", 0.0) + - baseline.get("game_thread_tick_p99_ms", 0.0), + "tick_max_delta_ms": candidate.get("game_thread_tick_max_ms", 0.0) + - baseline.get("game_thread_tick_max_ms", 0.0), + }) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--scenario", choices=("movement", "arrival"), required=True) + parser.add_argument("--design", choices=(DESIGN, NEAR_ONLY_DESIGN), default=DESIGN) + args = parser.parse_args() + if not EDITOR.is_file(): + raise SystemExit(f"missing Unreal editor: {EDITOR}") + output_name = args.scenario + ("-near-only" if args.design == NEAR_ONLY_DESIGN else "") + out = ROOT / f"Saved/Linux/l122/{output_name}" + out.mkdir(parents=True, exist_ok=True) + runs = [] + for repetition in range(1, 6): + order = ("baseline", args.design) if repetition % 2 else (args.design, "baseline") + for condition in order: + runs.append(run(args.scenario, condition, repetition, args.design)) + (out / "runs-partial.json").write_text(json.dumps(runs, indent=2) + "\n", + encoding="utf-8") + pairs = pairs_for(runs, args.scenario, args.design) + summary = { + "design_id": args.design, + "design": ("Allow fast-travel near LOD0-2 admission after the 3x3 walkable ring is ready " + "while other expanded critical tiles remain unresident; skip directional sort " + "and far admission until the critical set is ready." + if args.design == NEAR_ONLY_DESIGN else + "Allow fast-travel near/far admissions after the 3x3 walkable ring is ready, " + "even while other expanded critical tiles remain unresident; critical requests " + "still submit first."), + "scenario": args.scenario, + "runs": runs, + "pairs": pairs, + "paired_median_near_ready_delta_pct": statistics.median( + pair["near_ready_delta_pct"] for pair in pairs), + "paired_median_far_holes_delta_pct": statistics.median( + pair["far_holes_delta_pct"] for pair in pairs), + "paired_median_tick_p99_delta_ms": statistics.median( + pair["tick_p99_delta_ms"] for pair in pairs), + "paired_median_tick_max_delta_ms": statistics.median( + pair["tick_max_delta_ms"] for pair in pairs), + } + (out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux122/l122-trace.patch b/Standalone/Research/WorldAssemblerM3/Linux122/l122-trace.patch new file mode 100644 index 0000000..197f207 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux122/l122-trace.patch @@ -0,0 +1,218 @@ +diff --git a/Source/VoxelForge/Private/VoxelWorld.cpp b/Source/VoxelForge/Private/VoxelWorld.cpp +index 42ac6f5..5c436ee 100644 +--- a/Source/VoxelForge/Private/VoxelWorld.cpp ++++ b/Source/VoxelForge/Private/VoxelWorld.cpp +@@ -195,10 +195,13 @@ public: + std::atomic bShuttingDown{false}; + std::atomic NextEditSequence{1}; + std::atomic RegionCompileStartCycles{0}; ++ std::atomic RegionCompileWorkerStartCycles{0}; + std::atomic RegionCompileEndCycles{0}; + std::atomic RegionCompileTileCount{0}; + std::atomic RegionCompileSerial{0}; ++ std::atomic RegionCompileWorkerThreadId{0}; + std::atomic RegionCompileWorkerSeconds{0.0}; ++ std::atomic RegionCompileWallSeconds{0.0}; + int32 FullResolutionLodCount = 0; + FString InitialFailureReason; + }; +@@ -11381,20 +11384,25 @@ void AVoxelWorld::StartPendingCoreRegionCompile() + 1, std::memory_order_relaxed) + 1; + State->RegionCompileStartCycles.store(ActiveCoreRegionCompileStartCycles, + std::memory_order_release); ++ State->RegionCompileWorkerStartCycles.store(0, std::memory_order_release); + State->RegionCompileEndCycles.store(0, std::memory_order_release); + State->RegionCompileTileCount.store(0, std::memory_order_release); ++ State->RegionCompileWorkerThreadId.store(0, std::memory_order_release); + State->RegionCompileWorkerSeconds.store(0.0, std::memory_order_release); ++ State->RegionCompileWallSeconds.store(0.0, std::memory_order_release); + if (VoxelForgeStartupTrace::IsActive()) + { + VoxelForgeStartupTrace::RecordEvent(TEXT("core_region_compile_start"), FString::Printf( + TEXT("\"compile_serial\":%d,\"request_serial\":%d,\"center\":[%d,%d,%d]," + "\"missing_region_tiles\":%d,\"retained_tiles\":%d,\"concurrent_admission\":%d," +- "\"active_tile_tasks\":%d,\"active_task_guard\":%d"), ++ "\"active_tile_tasks\":%d,\"active_generation_workers\":%d," ++ "\"active_task_guard\":%d,\"game_thread_id\":%u"), + CompileSerial, CoreRegionCompileRequestSerial, + CompileCenter.X, CompileCenter.Y, CompileCenter.Z, + MissingTiles.Num(), RetainedTiles.Num(), bAllowConcurrentGeneration ? 1 : 0, + ActiveTileTaskCount.load(std::memory_order_relaxed), +- ActiveTaskCount.load(std::memory_order_relaxed))); ++ ActiveGenerationWorkerCount.load(std::memory_order_relaxed), ++ ActiveTaskCount.load(std::memory_order_relaxed), FPlatformTLS::GetCurrentThreadId())); + } + ActiveTaskCount.fetch_add(1, std::memory_order_relaxed); + UE::Tasks::Launch(TEXT("VoxelForgeCoreTileRegion"), +@@ -11407,6 +11415,11 @@ void AVoxelWorld::StartPendingCoreRegionCompile() + ~FTaskGuard() { Counter.fetch_sub(1, std::memory_order_release); } + } Guard{ActiveTaskCount}; + ++ State->RegionCompileWorkerStartCycles.store(FPlatformTime::Cycles64(), ++ std::memory_order_release); ++ State->RegionCompileWorkerThreadId.store(FPlatformTLS::GetCurrentThreadId(), ++ std::memory_order_release); ++ + if (bShuttingDown.load(std::memory_order_acquire) + || State->bShuttingDown.load(std::memory_order_acquire)) + { +@@ -11458,6 +11471,8 @@ void AVoxelWorld::StartPendingCoreRegionCompile() + } + State->RegionCompileWorkerSeconds.store(CompileStats.workerSeconds, + std::memory_order_release); ++ State->RegionCompileWallSeconds.store(CompileStats.wallSeconds, ++ std::memory_order_release); + State->bRegionCompileSucceeded.store(bCompiled, std::memory_order_release); + if (!bCompiled) + { +@@ -12345,9 +12360,17 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + State->RegionCompileStartCycles.load(std::memory_order_acquire); + const uint64 CompileEndCycles = + State->RegionCompileEndCycles.load(std::memory_order_acquire); ++ const uint64 CompileWorkerStartCycles = ++ State->RegionCompileWorkerStartCycles.load(std::memory_order_acquire); + const double CompileElapsedMilliseconds = CompileEndCycles >= CompileStartCycles + ? FPlatformTime::ToSeconds64(CompileEndCycles - CompileStartCycles) * 1000.0 + : 0.0; ++ const double CompileQueueMilliseconds = CompileWorkerStartCycles >= CompileStartCycles ++ ? FPlatformTime::ToSeconds64(CompileWorkerStartCycles - CompileStartCycles) * 1000.0 ++ : 0.0; ++ const double CompileTaskMilliseconds = CompileEndCycles >= CompileWorkerStartCycles ++ ? FPlatformTime::ToSeconds64(CompileEndCycles - CompileWorkerStartCycles) * 1000.0 ++ : 0.0; + const double GateElapsedMilliseconds = CoreRegionPendingSinceSeconds > 0.0 + ? FMath::Max(0.0, FPlatformTime::Seconds() - CoreRegionPendingSinceSeconds) * 1000.0 + : 0.0; +@@ -12355,6 +12378,10 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + State->RegionCompileTileCount.load(std::memory_order_acquire); + const double CompileWorkerSeconds = + State->RegionCompileWorkerSeconds.load(std::memory_order_acquire); ++ const double CompileCoreWallSeconds = ++ State->RegionCompileWallSeconds.load(std::memory_order_acquire); ++ const uint32 CompileWorkerThreadId = ++ State->RegionCompileWorkerThreadId.load(std::memory_order_acquire); + const int32 CompileSerial = + State->RegionCompileSerial.load(std::memory_order_acquire); + const uint64 IndependentSubmissions = CoreRegionIndependentSubmissions; +@@ -12364,14 +12391,20 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + VoxelForgeStartupTrace::RecordEvent(TEXT("core_region_compile_complete"), FString::Printf( + TEXT("\"compile_serial\":%d,\"request_serial\":%d,\"center\":[%d,%d,%d]," + "\"success\":%d,\"compiled_tiles\":%d,\"compile_elapsed_ms\":%s," +- "\"compile_worker_s\":%s,\"pending_gate_elapsed_ms\":%s," ++ "\"compile_queue_ms\":%s,\"compile_task_ms\":%s," ++ "\"compile_core_wall_ms\":%s,\"compile_worker_s\":%s," ++ "\"compile_worker_thread_id\":%u,\"pending_gate_elapsed_ms\":%s," + "\"independent_submissions\":%llu," + "\"dependent_declines\":%llu,\"concurrent_admission\":%d"), + CompileSerial, ActiveCoreRegionCompileRequestSerial, + ActiveCoreRegionCompileCenter.X, ActiveCoreRegionCompileCenter.Y, + ActiveCoreRegionCompileCenter.Z, bRegionCompileSucceeded ? 1 : 0, + CompiledTileCount, *FString::SanitizeFloat(CompileElapsedMilliseconds), ++ *FString::SanitizeFloat(CompileQueueMilliseconds), ++ *FString::SanitizeFloat(CompileTaskMilliseconds), ++ *FString::SanitizeFloat(CompileCoreWallSeconds * 1000.0), + *FString::SanitizeFloat(CompileWorkerSeconds), ++ CompileWorkerThreadId, + *FString::SanitizeFloat(GateElapsedMilliseconds), + static_cast(IndependentSubmissions), + static_cast(DependentDeclines), +@@ -12584,6 +12617,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + && bWalkableRingReadyForReserve; + int32 SchedulerSubmittedByLevel[9] = {}; + int32 SchedulerLoadAttemptsByLevel[9] = {}; ++ int32 SchedulerRegionCompileDeclinesByLevel[9] = {}; ++ int32 SchedulerFarParentDeclinesByLevel[9] = {}; + int32 SchedulerLevelCapReturns[9] = {}; + int32 SchedulerLevelCapBreaks[9] = {}; + int32 SchedulerCriticalFullCapBreaks = 0; +@@ -12951,7 +12986,21 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + if(CoreState.IsValid() + &&CoreState->bFarWorldRequired.load(std::memory_order_acquire) + &&!CoreState->bFarWorldReady.load(std::memory_order_acquire)) ++ { ++ if (bTraceStreamingScheduler && Tile.Level >= 0 && Tile.Level < 9) ++ { ++ ++SchedulerFarParentDeclinesByLevel[Tile.Level]; ++ } + return; ++ } ++ } ++ if (bTraceStreamingScheduler && bUseVoxelForgeCore && bCoreRegionCompilePending ++ && GVoxelForgeCoreRegionConcurrentAdmission != 0 ++ && (PendingCoreRegionTileSet.Contains(Tile) ++ || CoreRegionCompileTilesInFlight.Contains(Tile)) ++ && Tile.Level >= 0 && Tile.Level < 9) ++ { ++ ++SchedulerRegionCompileDeclinesByLevel[Tile.Level]; + } + if (Tile.Level >= 0 && Tile.Level < 9) + { +@@ -13661,10 +13710,37 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + const double BusySeconds = FPlatformTime::ToSeconds64(WorkerCyclesDelta); + const double SchedulerUpdateMilliseconds = (SchedulerTraceNow + - StreamingUpdateStartSeconds) * 1000.0; ++ int32 CriticalMissingTiles = 0; ++ for (const FVoxelTileKey& Tile : CriticalDesiredTiles) ++ { ++ if (!LoadedTiles.Contains(Tile) && !CollisionNotRequiredTiles.Contains(Tile)) ++ { ++ ++CriticalMissingTiles; ++ } ++ } ++ int32 CollisionGateMissingTiles = 0; ++ for (const FVoxelTileKey& Tile : CollisionGateDesiredTiles) ++ { ++ if (!LoadedTiles.Contains(Tile) && !CollisionNotRequiredTiles.Contains(Tile)) ++ { ++ ++CollisionGateMissingTiles; ++ } ++ } ++ const FIntVector DesiredCenterLag = DesiredEpochCenterChunk - CenterChunk; ++ const TSharedPtr TraceCoreState = ++ VoxelForgeCoreState; ++ const bool bFarWorldRequiredForTrace = TraceCoreState.IsValid() ++ && TraceCoreState->bFarWorldRequired.load(std::memory_order_relaxed); ++ const bool bFarWorldReadyForTrace = TraceCoreState.IsValid() ++ && TraceCoreState->bFarWorldReady.load(std::memory_order_relaxed); + VoxelForgeStartupTrace::RecordEvent(TEXT("scheduler_admission"), FString::Printf( + TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"desired_rebuilt\":%d," + "\"submission_loop_ran\":%d,\"all_chunks_loaded\":%d," + "\"walkable_ready\":%d,\"moving\":%d," ++ "\"pawn_collision_gate_engaged\":%d,\"critical_tiles_ready\":%d," ++ "\"critical_missing\":%d,\"collision_gate_missing\":%d," ++ "\"desired_center_lag\":[%d,%d,%d],\"desired_center_current\":%d," ++ "\"far_world_required\":%d,\"far_world_ready\":%d," + "\"core_region_compile_pending\":%d,\"core_ready\":%d,\"coarse_first\":%d," + "\"dynamic_near_reserve\":%d,\"legacy_pending_result_cap\":%d," + "\"core_region_concurrent_admission\":%d," +@@ -13685,6 +13761,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + "\"level_candidate_scan_ms\":%s,\"passage_score_ms\":%s," + "\"level_sort_ms\":%s,\"level_admit_ms\":%s," + "\"desired_missing_by_lod\":%s,\"loaded_by_lod\":%s,\"pending_by_lod\":%s," ++ "\"region_compile_declines_by_lod\":%s," ++ "\"far_parent_declines_by_lod\":%s," + "\"submitted_by_lod\":%s,\"load_attempts_by_lod\":%s," + "\"admission_cap_by_lod\":%s,\"declined_rule_by_lod\":%s," + "\"level_cap_returns_by_lod\":%s,\"level_cap_breaks_by_lod\":%s," +@@ -13695,6 +13773,11 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + static_cast(GFrameCounter), CenterChunk.X, CenterChunk.Y, CenterChunk.Z, + bDesiredRebuiltThisUpdate ? 1 : 0, bSubmissionLoopRan ? 1 : 0, bAllChunksLoaded ? 1 : 0, + bWalkableRingReadyForReserve ? 1 : 0, bPawnMoving ? 1 : 0, ++ bPawnCollisionGateEngaged ? 1 : 0, CriticalMissingTiles == 0 ? 1 : 0, ++ CriticalMissingTiles, CollisionGateMissingTiles, ++ DesiredCenterLag.X, DesiredCenterLag.Y, DesiredCenterLag.Z, ++ DesiredCenterLag == FIntVector::ZeroValue ? 1 : 0, ++ bFarWorldRequiredForTrace ? 1 : 0, bFarWorldReadyForTrace ? 1 : 0, + bCoreRegionCompilePending ? 1 : 0, IsVoxelForgeCoreReady() ? 1 : 0, + GVoxelForgeStreamingCoarseFirst, GVoxelForgeStreamingDynamicNearReserve, + GVoxelForgeStreamingLegacyPendingResultCap, +@@ -13721,7 +13804,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw + *FString::SanitizeFloat(LevelSortMilliseconds), + *FString::SanitizeFloat(LevelAdmissionMilliseconds), + *IntArrayJson(DesiredMissingByLevel), *IntArrayJson(LoadedByLevel), +- *IntArrayJson(PendingByLevel), *IntArrayJson(SchedulerSubmittedByLevel), ++ *IntArrayJson(PendingByLevel), *IntArrayJson(SchedulerRegionCompileDeclinesByLevel), ++ *IntArrayJson(SchedulerFarParentDeclinesByLevel), *IntArrayJson(SchedulerSubmittedByLevel), + *IntArrayJson(SchedulerLoadAttemptsByLevel), *IntArrayJson(AdmissionCapByLevel), + *IntArrayJson(DeclinedRuleByLevel), *IntArrayJson(SchedulerLevelCapReturns), + *IntArrayJson(SchedulerLevelCapBreaks), FullCapDeclinedTiles, ReserveDeclinedTiles,