VoxelForge/Standalone/Research/WorldAssemblerM3/Linux124/RunL124WorkerDiagnostics.py

#!/usr/bin/env python3
"""Collect the separate worker-idle and admission timeline for L124's unchanged headless path."""
from __future__ import annotations

import json
import re
import subprocess
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[4]
DATA = ROOT / "Standalone/data/owner_world.json"
BIN = ROOT / "Saved/Linux/l124/bin/VoxelForgeCoreHeadless-l124-clang"
OUT = ROOT / "Saved/Linux/l124/workers"
STAMP_TOOL = ROOT / "Standalone/Tools/verify/source_stamp.py"
CAMERAS = {
    "sw": (8.0, 16.0, -372.2626953125),
    "fi": (25.8369140625, 3.19921875, -737.0),
    "tn": (19.166015625, -14.2001953125, -753.3740234375),
    "vs": (-20.0625, -5.068359375, -900.353515625),
}
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M)
COLD = re.compile(
    r"^\[ColdTimeline\] event=(\S+) edge=(\S+) at_ms=([0-9.]+) cpu_ms=([0-9.]+)$",
    re.M,
)
TIMELINE = re.compile(
    r"^\[CpuTileTimeline\] event=(\S+) key=\(([-0-9]+),([-0-9]+),([-0-9]+),lod(\d+)\) "
    r"at_ms=([0-9.-]+) frame=\d+ worker_id=(-?\d+) queue_depth=(\d+) "
    r"active_workers=(\d+) priority_rank=(\d+)", re.M,
)
PROFILE = re.compile(r"^\[ShapeProfile\] (.*)$", re.M)
BREAKDOWN = re.compile(r"^\[ShapeBreakdown\] (.*)$", re.M)
NEAR = re.compile(
    r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$",
    re.M,
)
PAIR = re.compile(r"([a-zA-Z0-9_+.-]+)=([0-9.]+)(?:ms)?")


def intervals(events: list[tuple[float, float]], start: float, end: float) -> float:
    return sum(max(0.0, min(finish, end) - max(begin, start))
               for begin, finish in events)


def run(stratum: str, workers: int, expected_stamp: str) -> dict:
    OUT.mkdir(parents=True, exist_ok=True)
    log = OUT / f"{stratum}-w{workers}.log"
    camera = ",".join(f"{value:.12f}" for value in CAMERAS[stratum]) + ",0,0"
    command = [
        "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
        "timeout", "1740s", str(BIN), str(DATA), "--world-assembler-prototype",
        "--stream-benchmark", "--stream-pipeline", f"--workers={workers}",
        "--clip-radius=3", "--max-clip-level=8", "--max-level=8",
        f"--camera={camera}", "--profile-lod-work", "--stream-scheduler-policy=3",
        "--cold-worker-timeline",
    ]
    started = time.monotonic()
    with log.open("w", encoding="utf-8") as output:
        result = subprocess.run(command, cwd=ROOT, stdout=output,
                                stderr=subprocess.STDOUT, check=False, timeout=1740)
    content = log.read_text(encoding="utf-8", errors="replace")
    stamp = STAMP.search(content)
    cold = {(event, edge): (float(at), float(cpu))
            for event, edge, at, cpu in COLD.findall(content)}
    timeline = [
        {"event": event, "key": (int(x), int(y), int(z), int(lod)),
         "at_ms": float(at), "worker_id": int(worker),
         "queue_depth": int(depth), "active_workers": int(active),
         "priority_rank": int(rank)}
        for event, x, y, z, lod, at, worker, depth, active, rank in TIMELINE.findall(content)
    ]
    base_begin, base_end = cold.get(("base_compile", "begin")), cold.get(("base_compile", "end"))
    near = NEAR.search(content)
    profile_match, breakdown_match = PROFILE.search(content), BREAKDOWN.search(content)
    if (result.returncode or stamp is None or stamp.group(1) != expected_stamp
            or base_begin is None or base_end is None or near is None
            or profile_match is None or breakdown_match is None):
        raise RuntimeError(f"worker diagnostic incomplete: {log}; exit={result.returncode}")
    begin_ms, begin_cpu = base_begin
    end_ms, end_cpu = base_end
    admitted = [event for event in timeline if event["event"] == "admitted" and event["key"][3] <= 2]
    post_compile = [event for event in admitted if event["at_ms"] >= end_ms]
    idle_by_worker: dict[int, list[tuple[float, float]]] = {}
    idle_begin: dict[int, float] = {}
    for event in timeline:
        worker = event["worker_id"]
        if worker < 0:
            continue
        if event["event"] == "idle_begin":
            idle_begin[worker] = event["at_ms"]
        elif event["event"] == "idle_end" and worker in idle_begin:
            idle_by_worker.setdefault(worker, []).append((idle_begin.pop(worker), event["at_ms"]))
    for worker, idle_start in idle_begin.items():
        idle_by_worker.setdefault(worker, []).append((idle_start, end_ms))
    idle_worker_ms = sum(intervals(spans, begin_ms, end_ms)
                         for spans in idle_by_worker.values())
    profile = {key: float(value) for key, value in PAIR.findall(profile_match.group(1))}
    breakdown = {key: float(value) for key, value in PAIR.findall(breakdown_match.group(1))}
    parallel_stages = {}
    for name in ("rock_lines", "candidate_axes", "candidate_merge", "detail_lines"):
        wall, worker = profile.get(f"{name}_wall"), profile.get(f"{name}_worker")
        if wall is not None and worker is not None and wall > 0.0:
            parallel_stages[name] = {
                "wall_ms": wall,
                "worker_ms": worker,
                "average_busy_workers": worker / wall,
            }
    row = {
        "stratum": stratum,
        "workers": workers,
        "exit_code": result.returncode,
        "wall_s": round(time.monotonic() - started, 3),
        "source_stamp": stamp.group(1),
        "base_compile_start_ms": begin_ms,
        "base_shape_ready_ms": end_ms,
        "base_compile_wall_ms": end_ms - begin_ms,
        "process_cpu_ms_during_compile": end_cpu - begin_cpu,
        "average_process_cores_during_compile": (end_cpu - begin_cpu) / max(1e-9, end_ms - begin_ms),
        "stream_workers_idle_ms_during_compile": idle_worker_ms,
        "stream_idle_worker_equivalents": idle_worker_ms / max(1e-9, end_ms - begin_ms),
        "first_lod0_2_admission_ms": min((event["at_ms"] for event in admitted), default=None),
        "first_lod0_2_shape_dependent_admission_ms": min(
            (event["at_ms"] for event in post_compile), default=None),
        "near_ready_s": float(near.group(3)),
        "lod0_2_admissions": len(admitted),
        "lod0_2_admissions_before_shape_ready": sum(event["at_ms"] < end_ms for event in admitted),
        "shape_parallel_stages": parallel_stages,
        "owner_shape_worker_ms": breakdown.get("owner_shape_worker"),
        "log": str(log.relative_to(ROOT)),
    }
    print("[L124WorkerRun] " + json.dumps(row, sort_keys=True), flush=True)
    return row


def main() -> int:
    expected_stamp = subprocess.run(
        ["python3", "-B", str(STAMP_TOOL), "stamp", "--root", str(ROOT)],
        cwd=ROOT, text=True, stdout=subprocess.PIPE, check=True,
    ).stdout.strip()
    rows = [run(stratum, workers, expected_stamp)
            for stratum in CAMERAS for workers in (8, 12)]
    (OUT / "summary.json").write_text(
        json.dumps({"source_stamp": expected_stamp, "runs": rows}, indent=2) + "\n",
        encoding="utf-8",
    )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())