VoxelForge/Standalone/Research/WorldAssemblerM3/Linux105/RunL105Hashes.py

#!/usr/bin/env python3
"""Compare L105 exact tile hashes with L104 across all strata, compilers and workers."""
from __future__ import annotations

import json
import re
import subprocess
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l105/hashes"
DATA = ROOT / "Standalone/data/owner_world.json"
OLD = ROOT / "Saved/Linux/l104/hashes"
BINARIES = {
    "clang": ROOT / "Saved/Linux/l105/bin/VoxelForgeCoreHeadless-l105-clang",
    "gcc": ROOT / "Saved/Linux/l105/bin/VoxelForgeCoreHeadless-l105-gcc",
}
CAMERAS = {
    "sw": ((8.0, 16.0, -372.2626953125), 2850),
    "fi": ((25.8369140625, 3.19921875, -737.0), 2249),
    "tn": ((19.166015625, -14.2001953125, -753.3740234375), 2233),
    "vs": ((-20.0625, -5.068359375, -900.353515625), 2329),
}
TILE = re.compile(
    r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
    r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) "
    r"topology=(0x[0-9a-f]+)$", re.M)
WHOLE = re.compile(
    r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) "
    r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M)
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M)


def parse(path: Path) -> tuple[dict, tuple[str, ...], str]:
    content = path.read_text(encoding="utf-8", errors="replace")
    tiles = {(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))):
             m.group(5, 6, 7, 8) for m in TILE.finditer(content)}
    whole, stamp = WHOLE.search(content), STAMP.search(content)
    if whole is None or stamp is None or len(tiles) != int(whole.group(1)):
        raise RuntimeError(f"incomplete hash output: {path}")
    return tiles, whole.groups(), stamp.group(1)


def run(stratum: str, compiler: str, workers: int) -> dict:
    OUT.mkdir(parents=True, exist_ok=True)
    log = OUT / f"{stratum}-{compiler}-w{workers}.log"
    camera, expected_tiles = CAMERAS[stratum]
    camera_arg = ",".join(f"{v:.12f}" for v in camera) + ",0,0"
    command = [
        "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s",
        str(BINARIES[compiler]), str(DATA), "--world-assembler-prototype",
        "--stream-benchmark", "--stream-pipeline", f"--workers={workers}",
        "--clip-radius=3", "--max-clip-level=8", "--max-level=8",
        f"--camera={camera_arg}", "--stream-hash-tiles",
    ]
    started = time.monotonic()
    with log.open("w", encoding="utf-8") as stream:
        process = subprocess.run(command, cwd=ROOT, stdout=stream,
                                 stderr=subprocess.STDOUT, check=False, timeout=1740)
    if process.returncode:
        raise RuntimeError(f"hash process exited {process.returncode}: {log}")
    tiles, whole, stamp = parse(log)
    if len(tiles) != expected_tiles:
        raise RuntimeError(f"expected {expected_tiles} tiles, got {len(tiles)}: {log}")
    old_log = OLD / f"{stratum}-{compiler}-w{workers}.log"
    old_tiles, old_whole, _old_stamp = parse(old_log)
    clang_reference, clang_whole, _ = parse(OLD / f"{stratum}-clang-w1.log")

    def difference(reference: dict, candidate: dict) -> tuple[int, int]:
        keys = set(reference) | set(candidate)
        mismatches = sum(reference.get(key) != candidate.get(key) for key in keys)
        return mismatches, len(keys)

    old_mismatches, old_keys = difference(old_tiles, tiles)
    cross_mismatches, cross_keys = difference(clang_reference, tiles)
    row = {
        "stratum": stratum, "compiler": compiler, "workers": workers,
        "source_stamp": stamp, "tiles": len(tiles),
        "l104_same_cell_tile_mismatches": old_mismatches,
        "l104_same_cell_key_count": old_keys,
        "l104_same_cell_whole_hash_match": whole == old_whole,
        "clang_w1_tile_mismatches": cross_mismatches,
        "clang_w1_key_count": cross_keys,
        "clang_w1_whole_hash_match": whole == clang_whole,
        "wall_s": round(time.monotonic() - started, 3),
        "log": str(log.relative_to(ROOT)),
    }
    print("[L105HashRun] " + json.dumps(row, sort_keys=True), flush=True)
    row["tile_hashes"] = {"|".join(map(str, key)): value for key, value in tiles.items()}
    return row


def main() -> int:
    for path in (*BINARIES.values(), DATA):
        if not path.is_file():
            raise SystemExit(f"missing input: {path}")
    rows = []
    for stratum in CAMERAS:
        for compiler in BINARIES:
            for workers in (1, 8, 12):
                rows.append(run(stratum, compiler, workers))
                (OUT / "runs-partial.json").write_text(
                    json.dumps(rows, indent=2) + "\n", encoding="utf-8")
    failures = [row for row in rows if row["l104_same_cell_tile_mismatches"]
                or not row["l104_same_cell_whole_hash_match"]
                or row["clang_w1_tile_mismatches"]
                or not row["clang_w1_whole_hash_match"]]
    summary = {"runs": [{key: value for key, value in row.items() if key != "tile_hashes"}
                        for row in rows], "failed_runs": len(failures)}
    (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n",
                                      encoding="utf-8")
    return 1 if failures else 0


if __name__ == "__main__":
    raise SystemExit(main())