VoxelForge/Standalone/Research/WorldAssemblerM3/Linux85/RunL85TileProfiles.py

#!/usr/bin/env python3
"""Interleaved L84/L85 W8 profiles for the FI/TN heavy-tile floor."""
from __future__ import annotations

import json
import re
import statistics
import subprocess
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l85/tile-profiles-final"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARIES = {
    "L84": ROOT / "Saved/Linux/l84/bin/VoxelForgeCoreHeadless-l84-clang19",
    "L85": ROOT / "Saved/Linux/l85/bin/VoxelForgeCoreHeadless-clang19",
}
STAMPS = {
    "L84": "c433a039b0255ce3c4794bf20313b8a5a11442feeb07b6555f585688949741a5",
    "L85": "2967bb8aaea5a7beac0cd0735c0c2e5cccd7b31a9622dd33395e81ff1e5f2589",
}
CAMERAS = {
    1: (25.8369140625, 3.19921875, -737.0),
    2: (19.166015625, -14.2001953125, -753.3740234375),
}
NAMES = {1: "FI", 2: "TN"}
EXPECTED_NEAR = {1: 1216, 2: 1344}
EXPECTED_WHOLE = {1: 2249, 2: 2233}
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
TIMELINE = re.compile(
    r"^\[CpuTileTimeline\] event=finish key=\(([-0-9]+),([-0-9]+),([-0-9]+),lod([0-9]+)\) (.*)$", re.M)
STAGES = re.compile(
    r"^\[CpuTileStages\] key=\(([-0-9]+),([-0-9]+),([-0-9]+),lod([0-9]+)\) (.*)$", re.M)


def run(strate: int, version: str, rep: int) -> dict:
    out_dir = OUT / version.lower()
    out_dir.mkdir(parents=True, exist_ok=True)
    log = out_dir / f"{NAMES[strate].lower()}-w8-r{rep}.log"
    camera = ",".join(f"{v:.12f}" for v in CAMERAS[strate]) + ",0,0"
    command = [
        "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
        "env", "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", "timeout", "1740s",
        str(BINARIES[version]), str(DATA), "--world-assembler-prototype",
        "--stream-benchmark", "--stream-pipeline", "--workers=8", "--clip-radius=3",
        "--max-clip-level=8", "--max-level=8", f"--camera={camera}",
        "--profile-lod-work", "--instrument-tile-drain", "--stream-scheduler-policy=3",
    ]
    started = time.monotonic()
    with log.open("w", encoding="utf-8") as stream:
        result = subprocess.run(command, cwd=ROOT, stdout=stream,
                                stderr=subprocess.STDOUT, check=False, timeout=1740)
    output = log.read_text(encoding="utf-8", errors="replace")
    source, near, pipeline = STAMP.search(output), NEAR.search(output), PIPELINE.search(output)
    if result.returncode or source is None or near is None or pipeline is None:
        raise RuntimeError(f"profile run failed: exit={result.returncode} {log}")
    if source.group(1) != STAMPS[version]:
        raise RuntimeError(f"source stamp mismatch in {log}: {source.group(1)}")
    if (int(near.group(1)), int(near.group(2))) != (EXPECTED_NEAR[strate], EXPECTED_NEAR[strate]):
        raise RuntimeError(f"incomplete LOD0-2 in {log}")
    fields = dict(FIELDS.findall(pipeline.group(1)))
    done, total = (int(value) for value in fields["whole_tiles"].split("/"))
    if (done, total) != (EXPECTED_WHOLE[strate], EXPECTED_WHOLE[strate]):
        raise RuntimeError(f"incomplete whole view in {log}: {done}/{total}")

    stages = {}
    for match in STAGES.finditer(output):
        key = tuple(int(value) for value in match.group(1, 2, 3, 4))
        stages[key] = dict(FIELDS.findall(match.group(5)))
    tiles = []
    for match in TIMELINE.finditer(output):
        key = tuple(int(value) for value in match.group(1, 2, 3, 4))
        if key[3] > 2:
            continue
        stage = stages.get(key, {})
        tile = dict(FIELDS.findall(match.group(5)))
        tiles.append({
            "key": key,
            "service_ms": float(tile["service_ms"]),
            "samples": int(stage.get("samples", "0")),
            "mc_grid_ms": float(stage.get("mc_grid_ms", "0")),
            "mc_grid_workers": int(stage.get("mc_grid_workers", "1")),
        })
    if len(tiles) != EXPECTED_NEAR[strate]:
        raise RuntimeError(f"incomplete near tile profile in {log}: {len(tiles)}")
    full = [tile for tile in tiles if tile["samples"] == 42875]
    slowest = max(full, key=lambda tile: tile["service_ms"])
    target = next((tile for tile in full if tile["key"] == (0, 1, -93, 0)), None)
    if target is None:
        raise RuntimeError(f"target junction tile missing from profile: {log}")
    row = {
        "version": version, "strate": strate, "name": NAMES[strate], "replicate": rep,
        "workers": 8, "lod02_ready_s": float(near.group(3)),
        "whole_view_ready_s": float(fields["whole_view_ready"].removesuffix("s")),
        "shape_s": float(fields["shape"].removesuffix("s")),
        "far_shape_s": float(fields["far_shape"].removesuffix("s")),
        "full_grid_tile_count": len(full), "slowest_full_grid_tile": slowest,
        "target_tile": target, "source_stamp": source.group(1),
        "log": str(log.relative_to(ROOT)),
        "wall_s": round(time.monotonic() - started, 3),
    }
    print("[L85TileProfile] " + json.dumps(row, sort_keys=True), flush=True)
    return row


def main() -> int:
    rows = []
    for strate in (1, 2):
        for rep in range(1, 4):
            order = ("L84", "L85") if rep % 2 else ("L85", "L84")
            for version in order:
                rows.append(run(strate, version, rep))
                (OUT / "runs-partial.json").write_text(
                    json.dumps(rows, indent=2) + "\n", encoding="utf-8")
    summary = []
    for strate in (1, 2):
        for version in ("L84", "L85"):
            subset = [row for row in rows
                      if row["strate"] == strate and row["version"] == version]
            item = {"strate": strate, "name": NAMES[strate], "version": version,
                    "runs": [row["log"] for row in subset]}
            for metric in ("lod02_ready_s", "whole_view_ready_s", "shape_s", "far_shape_s"):
                values = [row[metric] for row in subset]
                item[f"{metric}_runs"] = values
                item[f"{metric}_median"] = statistics.median(values)
            for metric in ("service_ms", "mc_grid_ms"):
                values = [row["slowest_full_grid_tile"][metric] for row in subset]
                item[f"slowest_full_grid_{metric}_runs"] = values
                item[f"slowest_full_grid_{metric}_median"] = statistics.median(values)
            item["slowest_full_grid_keys"] = [row["slowest_full_grid_tile"]["key"]
                                                for row in subset]
            item["slowest_full_grid_workers"] = [
                row["slowest_full_grid_tile"]["mc_grid_workers"] for row in subset]
            for metric in ("service_ms", "mc_grid_ms"):
                values = [row["target_tile"][metric] for row in subset]
                item[f"target_{metric}_runs"] = values
                item[f"target_{metric}_median"] = statistics.median(values)
            item["target_grid_workers"] = [row["target_tile"]["mc_grid_workers"]
                                            for row in subset]
            item["floor_s"] = item["shape_s_median"] + item["far_shape_s_median"] \
                + item["slowest_full_grid_service_ms_median"] / 1000.0
            summary.append(item)
    (OUT / "summary.json").write_text(
        json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n",
        encoding="utf-8")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())