#!/usr/bin/env python3 """Matched null-RHI FI/SW startup traces for Linux 94.""" from __future__ import annotations import argparse import json import re import subprocess import time from pathlib import Path ROOT = Path(__file__).resolve().parents[4] HOST = Path("/home/jahni/VFHost") HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l94" HOST_SAVED = HOST / "Saved/Linux/l94" EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd") OUT = ROOT / "Saved/Linux/l94/unreal-arrival" SPAWNS_M = { "FI": (25.8369140625, 3.19921875, -737.0), "SW": (25.8369140625, 3.19921875, -200.0), } MOVE_START_S = 20.0 MOVE_DURATION_S = 12.0 DWELL_S = 5.0 EXIT_S = MOVE_START_S + MOVE_DURATION_S + DWELL_S def read_events(path: Path) -> list[dict]: events: list[dict] = [] if path.is_file(): for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): if line.startswith("{"): try: events.append(json.loads(line)) except json.JSONDecodeError: pass return events def run(stratum: str, mode: str, repeat: int, phase: str) -> dict: slot = 1 if stratum == "FI" else 0 boost = 0 if mode == "off" else 1 label = f"{phase}-{stratum.lower()}-{mode}-{repeat}" trace = OUT / f"{label}.trace.jsonl" abslog = OUT / f"{label}.log" stdout = OUT / f"{label}.stdout.log" output = HOST_PLUGIN_SAVED / "output" / label userdir = HOST_SAVED / "user" / label zendata = HOST_SAVED / "zen" / label ddc = HOST_SAVED / "DDC" for directory in (OUT, output, userdir, zendata, ddc): directory.mkdir(parents=True, exist_ok=True) trace.unlink(missing_ok=True) spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWNS_M[stratum]) command = [ str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", "-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer", "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", "-archetype=TunnelNetwork", f"-slot={slot}", "-modes=export", "-gameconfig=1", "-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25", "-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0", "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0", f"-out={output}", "-voxel.StreamingCoarseFirst=1", "-voxel.StreamingDynamicNearReserve=1", "-voxel.StreamingLegacyPendingResultCap=0", "-voxel.CoreRegionConcurrentAdmission=1", "-voxel.StreamingResultBacklogCap=256", "-voxel.StreamingReadyQueueDepth=8", "-voxel.StreamingBackgroundWorkerCap=1", "-voxel.CoreColumnCacheDirectMapped=0", "-voxel.CoreMCGradientMemoization=1", "-voxel.StreamingCollisionLookahead=1", f"-voxel.StreamingStartupApplyBoost={boost}", "-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=140.0", f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}", f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}", f"-voxel.TestExitSeconds={EXIT_S:.1f}", "-voxel.TestMoveDirection=1,0,0", "-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0", "-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}", "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", f"-abslog={abslog}", ] wrapped = ["taskset", "-c", "0-11", "timeout", "1740s", *command] started = time.monotonic() with stdout.open("w", encoding="utf-8") as stream: result = subprocess.run(wrapped, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, check=False, timeout=1740) events = read_events(trace) samples = [e for e in events if e.get("type") == "streaming_sample"] arrival_samples = [e for e in samples if float(e.get("t_s", 0.0)) < MOVE_START_S] ready = None for event in arrival_samples: desired = sum(event.get("desired_by_lod", [0] * 9)[:3]) settled = sum(event.get("settled_by_lod", event.get("resident_by_lod", [0] * 9))[:3]) if desired and settled >= desired: ready = event break first_admission = next((e for e in events if e.get("type") == "scheduler_admission" and e.get("desired_rebuilt") == 1), None) stage_ends = {e.get("name"): e for e in events if e.get("type") == "stage_end"} core_events = {name: next((e for e in events if e.get("type") == name), None) for name in ("core_snapshot_serialization", "core_snapshot_from_json", "core_compile_launch", "core_compile_start", "core_bready")} begin_play = stage_ends.get("BeginPlay", {}) compile_start = core_events["core_compile_start"] core_bready = core_events["core_bready"] row = { "label": label, "stratum": stratum, "slot": slot, "mode": mode, "boost_cvar": boost, "repeat": repeat, "exit_code": result.returncode, "wall_s": round(time.monotonic() - started, 3), "sample_count": len(samples), "arrival_sample_count": len(arrival_samples), "first_lod02_full_ready_s": ready.get("t_s") if ready else None, "first_lod02_full_desired": sum(ready.get("desired_by_lod", [0] * 9)[:3]) if ready else None, "first_lod02_full_resident": sum(ready.get("resident_by_lod", [0] * 9)[:3]) if ready else None, "first_lod02_full_settled": sum(ready.get("settled_by_lod", ready.get("resident_by_lod", [0] * 9))[:3]) if ready else None, "first_admission_t_s": first_admission.get("t_s") if first_admission else None, "begin_play_end_t_s": begin_play.get("t_s"), "begin_play_duration_s": begin_play.get("duration_s"), "core_bready_t_s": core_bready.get("t_s") if core_bready else None, "compile_start_to_bready_s": round( core_bready["t_s"] - compile_start["t_s"], 6) if core_bready and compile_start else None, "startup_events": {key: value for key, value in core_events.items() if value}, "snapshot_serialize_ms": core_events["core_snapshot_serialization"].get("duration_ms") if core_events["core_snapshot_serialization"] else None, "snapshot_from_json_ms": core_events["core_snapshot_from_json"].get("duration_ms") if core_events["core_snapshot_from_json"] else None, "trace": str(trace.relative_to(ROOT)), "abslog": str(abslog.relative_to(ROOT)), "stdout": str(stdout.relative_to(ROOT)), } stdout_text = stdout.read_text(encoding="utf-8", errors="replace") failure = re.search(r"\[VoxelForgeCoreWorldTest\] failed: ([^\r\n]+)", stdout_text) if failure: row["world_test_failure"] = failure.group(1) apply_events = [e for e in events if e.get("type") == "startup_apply_budget"] row["apply_budget_events"] = apply_events with (OUT / "runs.jsonl").open("a", encoding="utf-8") as stream: stream.write(json.dumps(row, sort_keys=True) + "\n") print("[L94UnrealArrival] " + json.dumps(row, sort_keys=True), flush=True) if result.returncode != 0: raise SystemExit(f"Unreal commandlet failed: {stdout}; exit={result.returncode}") if not core_bready: raise SystemExit(f"Unreal startup trace has no bReady event: {trace}") if not ready or len(arrival_samples) < 8: raise SystemExit(f"Unreal arrival trace incomplete: {stdout}; exit={result.returncode}") return row def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--stratum", choices=("FI", "SW"), required=True) parser.add_argument("--mode", choices=("on", "off"), default="on") parser.add_argument("--repeat", type=int, default=1) parser.add_argument("--phase", choices=("before", "after", "after-settled", "after-hold20", "after-direct-json"), default="after") args = parser.parse_args() if not EDITOR.is_file(): raise SystemExit(f"missing Unreal commandlet executable: {EDITOR}") row = run(args.stratum, args.mode, args.repeat, args.phase) (OUT / f"{row['label']}.summary.json").write_text( json.dumps(row, indent=2) + "\n", encoding="utf-8") return 0 if __name__ == "__main__": raise SystemExit(main())