VoxelForge/Standalone/Research/WorldAssemblerM3/Linux175/RunL175CanonicalHashes.py

#!/usr/bin/env python3
"""Capture v241 canonical owner hashes across Clang/GCC and W1/W8/W12."""
from __future__ import annotations

import json
import re
import subprocess
import sys
import time
from pathlib import Path


ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l175/hash-cross-compiler"
DATASETS = {
    "owner4": ROOT / "Standalone/data/owner_world.json",
    "owner2": ROOT / "Standalone/data/regression/owner_world_2strate_l61.json",
}
RECIPES = {
    "owner4": ("surface-world",),
    "owner2": ("surface-world", "floating-islands"),
}
BINARIES = {
    "clang": ROOT / "Standalone/bin/linux/VoxelForgeCoreHeadless-l175-final-clang",
    "gcc": ROOT / "Standalone/bin/linux/VoxelForgeCoreHeadless-l175-final-gcc",
}
BASELINE_ROOT = ROOT / "Standalone/Tools/verify/baselines/owner-hash/canonical"
BASELINES = {
    ("owner2", "surface-world"): (
        "owner-hash-surface-world-style1-v240.baseline",
        "owner-hash-surface-world-style1-v241.baseline"),
    ("owner2", "floating-islands"): (
        "owner-hash-floating-islands-style2-v240.baseline",
        "owner-hash-floating-islands-style2-v241.baseline"),
}
CANONICAL_RE = re.compile(
    r"^\[Pipeline(?:FullDetailHash|WholeViewHash|LODHash|ExactQueryHash)\].*$", re.M)
ALL_HASH_RE = re.compile(
    r"^\[Pipeline(?:FullDetailHash|WholeViewHash|MaterialHash|LODHash|ExactQueryHash)\].*$", re.M)
STAMP_RE = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]+)$", re.M)


def run(dataset: str, recipe: str, compiler: str, workers: int,
        expected_stamp: str) -> dict:
    label = f"{dataset}-{recipe}-{compiler}-w{workers}"
    log = OUT / f"{label}.log"
    layout = OUT / f"{label}.layout.json"
    command = [
        "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
        "timeout", "1740s", str(BINARIES[compiler]), str(DATASETS[dataset]),
        "--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
        "--force-generate", f"--workers={workers}", "--clip-radius=3",
        "--max-clip-level=8", "--max-level=8", f"--recipe={recipe}",
        "--stream-hash", f"--resolved-layout-output={layout}",
    ]
    if dataset == "owner4":
        command.append("--seed=0")
    started = time.monotonic()
    try:
        with log.open("w", encoding="utf-8") as output:
            result = subprocess.run(command, cwd=ROOT, stdout=output,
                stderr=subprocess.STDOUT, check=False, timeout=1740)
        code: int | str = result.returncode
    except subprocess.TimeoutExpired:
        code = "OUTER_TIMEOUT"
    content = log.read_text(encoding="utf-8", errors="replace") if log.is_file() else ""
    hashes = ALL_HASH_RE.findall(content)
    canonical = CANONICAL_RE.findall(content)
    stamp = STAMP_RE.search(content)
    row = {
        "dataset": dataset, "recipe": recipe, "compiler": compiler,
        "workers": workers, "exit_code": code,
        "run_passed": code == 0 and len(canonical) == 12 and len(hashes) == 13
            and stamp is not None and stamp.group(1) == expected_stamp,
        "canonical_hashes": canonical, "all_pipeline_hashes": hashes,
        "source_stamp": stamp.group(1) if stamp else None,
        "elapsed_s": round(time.monotonic() - started, 3),
        "log": str(log.relative_to(ROOT)),
    }
    print("[L175CanonicalHashRun] " + json.dumps(row, sort_keys=True), flush=True)
    return row


def main() -> int:
    if any(not path.is_file() for path in (*DATASETS.values(), *BINARIES.values())):
        raise SystemExit("missing v241 owner input or final compiler binary")
    OUT.mkdir(parents=True, exist_ok=True)
    expected_stamp = subprocess.check_output([
        sys.executable, str(ROOT / "Standalone/Tools/verify/source_stamp.py"),
        "stamp", "--root", str(ROOT)], text=True).strip()
    rows = []
    groups = {}
    partial = OUT / "runs-partial.json"
    for dataset, recipes in RECIPES.items():
        for recipe in recipes:
            current = []
            for workers in (1, 8, 12):
                for compiler in ("clang", "gcc"):
                    row = run(dataset, recipe, compiler, workers, expected_stamp)
                    rows.append(row)
                    current.append(row)
                    partial.write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8")
            deterministic = (len(current) == 6 and all(row["run_passed"] for row in current)
                and all(row["all_pipeline_hashes"] == current[0]["all_pipeline_hashes"]
                    for row in current))
            group = f"{dataset}/{recipe}"
            groups[group] = {"all_compilers_workers_match": deterministic,
                "canonical_rows": len(current[0]["canonical_hashes"]) if current else 0,
                "pipeline_rows": len(current[0]["all_pipeline_hashes"]) if current else 0}
            print("[L175CanonicalHashGroup] " + json.dumps(
                {"group": group, **groups[group]}, sort_keys=True), flush=True)

    passed = len(rows) == 18 and all(row["run_passed"] for row in rows) \
        and all(group["all_compilers_workers_match"] for group in groups.values())
    if passed:
        for key, (old_name, new_name) in BASELINES.items():
            canonical = next(row["canonical_hashes"] for row in rows
                if (row["dataset"], row["recipe"], row["compiler"], row["workers"])
                    == (*key, "clang", 12))
            (BASELINE_ROOT / new_name).write_text("\n".join(canonical) + "\n",
                encoding="utf-8")
            old = CANONICAL_RE.findall((BASELINE_ROOT / old_name).read_text(
                encoding="utf-8", errors="replace"))
            groups[f"{key[0]}/{key[1]}"]["changed_v240_rows"] = sum(
                a != b for a, b in zip(old, canonical))
    summary = {"status": "PASS" if passed else "FAIL",
        "source_stamp": expected_stamp, "runs": rows, "groups": groups}
    (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n",
        encoding="utf-8")
    print(f"[L175CanonicalHashSummary] status={summary['status']} runs={len(rows)} "
        f"groups={len(groups)}", flush=True)
    return 0 if passed else 1


if __name__ == "__main__":
    raise SystemExit(main())