VoxelForge/Standalone/Research/WorldAssemblerM3/Linux95/RunL95UnrealArrival.py

#!/usr/bin/env python3
"""Run one pinned null-RHI SW/FI arrival for Linux 95."""
from __future__ import annotations

import argparse
import json
import re
import subprocess
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[4]
HOST = Path("/home/jahni/VFHost")
HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l95"
HOST_SAVED = HOST / "Saved/Linux/l95"
EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd")
OUT = ROOT / "Saved/Linux/l95/unreal-arrival"
SPAWNS_M = {
    "FI": (25.8369140625, 3.19921875, -737.0),
    "SW": (25.8369140625, 3.19921875, -200.0),
}
MOVE_START_S = 20.0
MOVE_DURATION_S = 12.0
EXIT_S = MOVE_START_S + MOVE_DURATION_S + 5.0


def read_events(path: Path) -> list[dict]:
    events = []
    if path.is_file():
        for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
            if line.startswith("{"):
                try:
                    events.append(json.loads(line))
                except json.JSONDecodeError:
                    pass
    return events


def run(stratum: str, repeat: int, phase: str, ready_depth: int,
        background_cap: int, startup_boost: int,
        snapshot_file: Path | None) -> dict:
    label = (f"{phase}-{stratum.lower()}-rd{ready_depth}-bg{background_cap}"
             f"-boost{startup_boost}-{repeat}")
    trace = OUT / f"{label}.trace.jsonl"
    abslog = OUT / f"{label}.log"
    stdout = OUT / f"{label}.stdout.log"
    output = HOST_PLUGIN_SAVED / "output" / label
    userdir = HOST_SAVED / "user" / label
    zendata = HOST_SAVED / "zen" / label
    ddc = HOST_SAVED / "DDC"
    for directory in (OUT, output, userdir, zendata, ddc):
        directory.mkdir(parents=True, exist_ok=True)
    trace.unlink(missing_ok=True)
    spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWNS_M[stratum])
    command = [
        str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended",
        "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch",
        "-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer",
        "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}",
        f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0",
        "-archetype=TunnelNetwork", f"-slot={1 if stratum == 'FI' else 0}",
        "-modes=export", "-gameconfig=1", "-coreworldtest=1",
        "-ownerfourstrateplan=1", "-corevoxelsizecm=25", "-coreislandtest=0",
        "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0",
        "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0",
        f"-out={output}", "-voxel.StreamingCoarseFirst=1",
        "-voxel.StreamingDynamicNearReserve=1",
        "-voxel.StreamingLegacyPendingResultCap=0",
        "-voxel.CoreRegionConcurrentAdmission=1",
        "-voxel.StreamingResultBacklogCap=256",
        f"-voxel.StreamingReadyQueueDepth={ready_depth}",
        f"-voxel.StreamingBackgroundWorkerCap={background_cap}",
        "-voxel.CoreColumnCacheDirectMapped=0", "-voxel.CoreMCGradientMemoization=1",
        "-voxel.StreamingCollisionLookahead=1",
        f"-voxel.StreamingStartupApplyBoost={startup_boost}",
        "-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=140.0",
        f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}",
        f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}",
        f"-voxel.TestExitSeconds={EXIT_S:.1f}", "-voxel.TestMoveDirection=1,0,0",
        "-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0",
        "-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}",
        "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1",
        "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}",
        f"-abslog={abslog}",
    ]
    if snapshot_file is not None:
        snapshot_file.parent.mkdir(parents=True, exist_ok=True)
        command.append(f"-voxel.StartupTraceCoreSnapshotFile={snapshot_file}")
    wrapped = ["taskset", "-c", "0-11", "timeout", "1740s", *command]
    started = time.monotonic()
    with stdout.open("w", encoding="utf-8") as stream:
        result = subprocess.run(wrapped, cwd=ROOT, stdout=stream,
                                stderr=subprocess.STDOUT, check=False, timeout=1740)

    events = read_events(trace)
    samples = [e for e in events if e.get("type") == "streaming_sample"]
    arrival = [e for e in samples if float(e.get("t_s", 0.0)) < MOVE_START_S]
    full = next((e for e in arrival
                 if sum(e.get("desired_by_lod", [0] * 9)[:3]) > 0
                 and sum(e.get("settled_by_lod", [0] * 9)[:3])
                 >= sum(e.get("desired_by_lod", [0] * 9)[:3])), None)
    first_admission = next((e for e in events if e.get("type") == "scheduler_admission"
                            and e.get("desired_rebuilt") == 1), None)
    request_events = [e for e in events if e.get("type") == "request" and e.get("epoch") == 0]
    tile_by_ordinal = {e.get("request_ordinal"): e for e in events
                       if e.get("type") == "tile"}
    collision_count = sum(1 for e in events if e.get("type") == "collision_ready"
                          and e.get("level", 9) <= 2
                          and full and e.get("t_s", 1e30) <= full["t_s"])
    initial_tiles = [tile_by_ordinal[r.get("request_ordinal")] for r in request_events
                     if full and r.get("t_s", 1e30) <= full["t_s"]
                     and r.get("request_ordinal") in tile_by_ordinal
                     and tile_by_ordinal[r["request_ordinal"]].get("level", 9) <= 2]
    near_frames = [e for e in events if e.get("type") == "apply_frame" and first_admission
                   and full and first_admission["t_s"] <= e.get("t_s", -1) <= full["t_s"]]
    near_sched = [e for e in events if e.get("type") == "scheduler_admission" and first_admission
                  and full and first_admission["t_s"] <= e.get("t_s", -1) <= full["t_s"]]
    near_requests = [r for r in request_events if full and r.get("t_s", 1e30) <= full["t_s"]
                     and r.get("level", 9) <= 2]
    max_req = max((e["t_s"] for e in near_requests), default=None)
    row = {
        "label": label, "stratum": stratum, "repeat": repeat, "phase": phase,
        "ready_queue_depth": ready_depth, "background_worker_cap": background_cap,
        "startup_apply_boost": startup_boost, "exit_code": result.returncode,
        "wall_s": round(time.monotonic() - started, 3),
        "desired_t_s": next((e.get("t_s") for e in events if e.get("type") == "desired_set"), None),
        "first_admission_t_s": first_admission.get("t_s") if first_admission else None,
        "lod02_full_settled_t_s": full.get("t_s") if full else None,
        "desired_by_lod": full.get("desired_by_lod", [])[:3] if full else None,
        "settled_by_lod": full.get("settled_by_lod", [])[:3] if full else None,
        "near_admitted_tiles": len(near_requests),
        "last_admission_t_s": max_req,
        "near_tile_events": len(initial_tiles),
        "near_worker_seconds": round(sum(e.get("worker_task_s", 0.0) for e in initial_tiles), 4),
        "near_core_seconds": round(sum(e.get("core_generate_s", 0.0) for e in initial_tiles), 4),
        "collision_ready_by_settle": collision_count,
        "burst_frame_count": len(near_frames),
        "burst_frame_ms_p95": round(sorted(e.get("frame_ms", 0.0) for e in near_frames)[
            min(len(near_frames) - 1, int(.95 * (len(near_frames) - 1)))], 3)
            if near_frames else None,
        "burst_frame_ms_max": round(max((e.get("frame_ms", 0.0) for e in near_frames), default=0.0), 3),
        "burst_max_meshes_per_frame": max((e.get("meshes_applied", 0) for e in near_frames), default=0),
        "burst_max_apply_ms": round(max((e.get("apply_ms", 0.0) for e in near_frames), default=0.0), 3),
        "burst_max_apply_budget_ms": max((e.get("apply_budget_ms", 0.0) for e in near_frames), default=0.0),
        "burst_max_admit_per_frame": max((sum(e.get("submitted_by_lod", [0] * 9)[:3])
                                            for e in near_sched), default=0),
        "trace": str(trace.relative_to(ROOT)), "abslog": str(abslog.relative_to(ROOT)),
        "stdout": str(stdout.relative_to(ROOT)),
    }
    stdout_text = stdout.read_text(encoding="utf-8", errors="replace")
    failure = re.search(r"\[VoxelForgeCoreWorldTest\] failed: ([^\r\n]+)", stdout_text)
    if failure:
        row["world_test_failure"] = failure.group(1)
    with (OUT / "runs.jsonl").open("a", encoding="utf-8") as stream:
        stream.write(json.dumps(row, sort_keys=True) + "\n")
    (OUT / f"{label}.summary.json").write_text(json.dumps(row, indent=2) + "\n",
                                                 encoding="utf-8")
    print("[L95UnrealArrival] " + json.dumps(row, sort_keys=True), flush=True)
    if result.returncode:
        raise SystemExit(f"Unreal commandlet failed: {stdout}; exit={result.returncode}")
    if not full or len(arrival) < 8:
        raise SystemExit(f"Unreal arrival trace incomplete: {stdout}; exit={result.returncode}")
    return row


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--stratum", choices=("FI", "SW"), required=True)
    parser.add_argument("--repeat", type=int, default=1)
    parser.add_argument("--phase", choices=("probe", "before", "after", "snapshot"), default="probe")
    parser.add_argument("--ready-depth", type=int, default=64)
    parser.add_argument("--background-cap", type=int, choices=(0, 1), default=1)
    parser.add_argument("--startup-boost", type=int, choices=(0, 1), default=1)
    parser.add_argument("--snapshot-file", type=Path)
    args = parser.parse_args()
    if not EDITOR.is_file():
        raise SystemExit(f"missing Unreal commandlet executable: {EDITOR}")
    run(args.stratum, args.repeat, args.phase, args.ready_depth,
        args.background_cap, args.startup_boost,
        args.snapshot_file.resolve() if args.snapshot_file else None)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())