VoxelForge/Standalone/Research/WorldAssemblerM3/Linux76/RunHashMatrixL76.py

#!/usr/bin/env python3
"""Compare exact per-tile hashes for L75 reference and L76 on Clang/GCC W1/W8/W12."""
from __future__ import annotations

import json
import re
import subprocess
import time
from pathlib import Path


ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l76/hashes"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARIES = {
    "clang": {
        "before": ROOT / "Saved/Linux/l75/bin/VoxelForgeCoreHeadless-l75-clang",
        "after": ROOT / "Standalone/bin/linux/VoxelForgeCoreHeadless-l76-clang",
    },
    "gcc": {
        "before": ROOT / "Saved/Linux/l75/bin-gcc/VoxelForgeCoreHeadless-l75-gcc",
        "after": ROOT / "Standalone/bin/linux/VoxelForgeCoreHeadless-l76-gcc",
    },
}
CAMERAS = {
    0: (8.0, 16.0, -372.2626953125),
    1: (25.8369140625, 3.19921875, -737.0),
    2: (19.166015625, -14.2001953125, -753.3740234375),
    3: (-20.0625, -5.068359375, -900.353515625),
}
HASH = re.compile(
    r"\[PipelineTileHash\] lod=(\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
    r"density=(0x[0-9a-fA-F]+) mesh=(0x[0-9a-fA-F]+) "
    r"payload=(0x[0-9a-fA-F]+) topology=(0x[0-9a-fA-F]+)"
)
PAIR = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")


def read_hashes(path: Path) -> dict[tuple[int, int, int, int], dict[str, str]]:
    result = {}
    with path.open(encoding="utf-8", errors="replace") as stream:
        for line in stream:
            match = HASH.search(line)
            if match:
                lod, x, y, z = (int(match.group(index)) for index in range(1, 5))
                result[(lod, x, y, z)] = {
                    "density": match.group(5).lower(), "mesh": match.group(6).lower(),
                    "payload": match.group(7).lower(), "topology": match.group(8).lower(),
                }
    return result


def run(compiler: str, workers: int, strate: int, mode: str) -> dict:
    folder = OUT / compiler / f"w{workers}"
    folder.mkdir(parents=True, exist_ok=True)
    log = folder / f"strate{strate}-{mode}.log"
    camera = ",".join(f"{value:.12f}" for value in CAMERAS[strate]) + ",0,0"
    command = [
        "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout",
        "1740s", str(BINARIES[compiler][mode]), str(DATA), "--world-assembler-prototype",
        "--stream-benchmark", "--stream-pipeline", "--force-generate",
        f"--workers={workers}", "--clip-radius=3", "--max-clip-level=8",
        "--max-level=8", f"--camera={camera}", "--stream-hash-tiles",
    ]
    started = time.monotonic()
    with log.open("w", encoding="utf-8") as output:
        result = subprocess.run(command, cwd=ROOT, stdout=output,
                                stderr=subprocess.STDOUT, check=False, timeout=1740)
    hashes = read_hashes(log)
    if result.returncode != 0 or not hashes:
        raise SystemExit(f"hash generation failed or no tile hashes; see {log}")
    row = {
        "compiler": compiler, "workers": workers, "strate": strate, "mode": mode,
        "exit_code": result.returncode, "wall_s": round(time.monotonic() - started, 3),
        "tile_hashes": len(hashes), "log": str(log.relative_to(ROOT)),
    }
    print("[L76HashRun] " + json.dumps(row, sort_keys=True), flush=True)
    return row


def compare(reference: Path, candidate: Path) -> dict:
    expected, actual = read_hashes(reference), read_hashes(candidate)
    missing, extra = expected.keys() - actual.keys(), actual.keys() - expected.keys()
    mismatches = {name: 0 for name in ("density", "mesh", "payload", "topology")}
    for key in expected.keys() & actual.keys():
        for name in mismatches:
            if expected[key][name] != actual[key][name]:
                mismatches[name] += 1
    return {
        "reference": str(reference.relative_to(ROOT)),
        "candidate": str(candidate.relative_to(ROOT)),
        "reference_tiles": len(expected), "candidate_tiles": len(actual),
        "missing_keys": len(missing), "extra_keys": len(extra),
        "mismatches": mismatches,
        "equal": not missing and not extra and not any(mismatches.values()),
    }


def main() -> int:
    missing = [str(path) for compiler in BINARIES.values()
               for path in compiler.values() if not path.is_file()]
    if not DATA.is_file():
        missing.append(str(DATA))
    if missing:
        raise SystemExit("missing input: " + ", ".join(missing))
    rows, comparisons = [], []
    for compiler in BINARIES:
        for workers in (1, 8, 12):
            for strate in range(4):
                before = run(compiler, workers, strate, "before")
                after = run(compiler, workers, strate, "after")
                rows.extend((before, after))
                comparison = compare(ROOT / before["log"], ROOT / after["log"])
                comparison.update(compiler=compiler, workers=workers, strate=strate)
                comparisons.append(comparison)
                print("[L76HashCompare] " + json.dumps(comparison, sort_keys=True), flush=True)
                if not comparison["equal"]:
                    raise SystemExit(f"hash mismatch; see {comparison['candidate']}")
    result = {"runs": rows, "comparisons": comparisons,
              "equal": all(row["equal"] for row in comparisons)}
    (OUT / "matrix-summary.json").write_text(json.dumps(result, indent=2) + "\n",
                                               encoding="utf-8")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())