VoxelForge/Standalone/Research/WorldAssemblerM3/Linux100/RunL100Hashes.py

#!/usr/bin/env python3
"""Check L100 whole-view tile hashes across W1/W8/W12 and Clang/GCC."""
from __future__ import annotations

import json
import re
import subprocess
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l100/hashes"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARIES = {
    "Clang": ROOT / "Saved/Linux/l100/bin/VoxelForgeCoreHeadless-l100-clang",
    "GCC": ROOT / "Saved/Linux/l100/bin/VoxelForgeCoreHeadless-l100-gcc",
}
CAMERAS = {
    "SW": (8.0, 16.0, -372.2626953125),
    "FI": (25.8369140625, 3.19921875, -737.0),
    "TN": (19.166015625, -14.2001953125, -753.3740234375),
    "VS": (-20.0625, -5.068359375, -900.353515625),
}
TILE_HASH = re.compile(
    r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
    r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) "
    r"topology=(0x[0-9a-f]+)$", re.M)
WHOLE_HASH = re.compile(
    r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) "
    r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M)
FIELDS = ("density", "mesh", "payload", "topology")


def run(strate: str, compiler: str, workers: int) -> dict:
    OUT.mkdir(parents=True, exist_ok=True)
    log = OUT / f"{strate.lower()}-{compiler.lower()}-w{workers}.log"
    camera = ",".join(f"{value:.12f}" for value in CAMERAS[strate]) + ",0,0"
    command = [
        "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
        "timeout", "1740s", str(BINARIES[compiler]), str(DATA),
        "--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
        f"--workers={workers}", "--clip-radius=3", "--max-clip-level=8",
        "--max-level=8", f"--camera={camera}", "--stream-hash-tiles",
    ]
    start = time.monotonic()
    with log.open("w", encoding="utf-8") as stream:
        result = subprocess.run(command, cwd=ROOT, stdout=stream,
                                stderr=subprocess.STDOUT, check=False, timeout=1740)
    output = log.read_text(encoding="utf-8", errors="replace")
    hashes = {
        (int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))):
        m.group(5, 6, 7, 8) for m in TILE_HASH.finditer(output)
    }
    whole = WHOLE_HASH.search(output)
    stamp = re.search(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", output, re.M)
    if (result.returncode or whole is None or stamp is None
            or len(hashes) != int(whole.group(1))):
        raise RuntimeError(f"hash run failed or incomplete: exit={result.returncode} {log}")
    row = {
        "strate": strate, "compiler": compiler, "workers": workers,
        "source_stamp": stamp.group(1), "exit_code": result.returncode,
        "wall_s": round(time.monotonic() - start, 3), "tile_count": len(hashes),
        "tiles_by_lod": {str(lod): sum(1 for key in hashes if key[0] == lod)
                         for lod in sorted({key[0] for key in hashes})},
        "whole_hash": list(whole.groups()), "tile_hashes": hashes,
        "log": str(log.relative_to(ROOT)),
    }
    print("[L100HashRun] " + json.dumps(
        {key: value for key, value in row.items() if key != "tile_hashes"},
        sort_keys=True), flush=True)
    return row


def compare(strate: str, reference: dict, candidate: dict) -> dict:
    keys = set(reference["tile_hashes"]) | set(candidate["tile_hashes"])
    mismatches = {
        field: sum(reference["tile_hashes"].get(key, (None,) * 4)[index]
                   != candidate["tile_hashes"].get(key, (None,) * 4)[index]
                   for key in keys)
        for index, field in enumerate(FIELDS)
    }
    row = {
        "strate": strate, "reference": f"{reference['compiler']}-w{reference['workers']}",
        "candidate": f"{candidate['compiler']}-w{candidate['workers']}",
        "tile_counts_match": reference["tile_count"] == candidate["tile_count"],
        "whole_hashes_match": reference["whole_hash"] == candidate["whole_hash"],
        "tile_hashes_match": all(value == 0 for value in mismatches.values())
            and reference["tile_hashes"].keys() == candidate["tile_hashes"].keys(),
        "component_mismatch_tiles": mismatches,
        "tiles_reference": reference["tile_count"], "tiles_candidate": candidate["tile_count"],
        "reference_whole_hash": reference["whole_hash"],
        "candidate_whole_hash": candidate["whole_hash"],
    }
    print("[L100HashComparison] " + json.dumps(row, sort_keys=True), flush=True)
    return row


def main() -> int:
    for binary in (*BINARIES.values(), DATA):
        if not binary.is_file():
            raise SystemExit(f"missing input: {binary}")
    detail, comparisons = [], []
    for strate in CAMERAS:
        group = {(compiler, workers): run(strate, compiler, workers)
                 for compiler in BINARIES for workers in (1, 8, 12)}
        reference = group[("Clang", 1)]
        for key, candidate in group.items():
            if key == ("Clang", 1):
                continue
            comparisons.append(compare(strate, reference, candidate))
        detail.extend({key: value for key, value in row.items() if key != "tile_hashes"}
                      for row in group.values())
        (OUT / "runs-partial.json").write_text(json.dumps(
            {"runs": detail, "comparisons": comparisons}, indent=2) + "\n",
            encoding="utf-8")
    summary = {"runs": detail, "comparisons": comparisons}
    (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n",
                                      encoding="utf-8")
    return 0 if all(row["tile_counts_match"] and row["whole_hashes_match"]
                    and row["tile_hashes_match"] for row in comparisons) else 1


if __name__ == "__main__":
    raise SystemExit(main())