#!/usr/bin/env python3 """Check Clang/GCC hash identity at W1/W8/W12 and repin v217 owner hashes.""" from __future__ import annotations import json import re import subprocess import time from pathlib import Path ROOT = Path(__file__).resolve().parents[4] OUT = ROOT / "Saved/Linux/l109/hashes" DATASETS = { "owner4": ROOT / "Standalone/data/owner_world.json", "owner2": ROOT / "Standalone/data/regression/owner_world_2strate_l61.json", } BINARIES = { "clang": ROOT / "Saved/Linux/l109/bin-final/VoxelForgeCoreHeadless-l109-clang", "gcc": ROOT / "Saved/Linux/l109/bin-final/VoxelForgeCoreHeadless-l109-gcc", } HASH = re.compile(r"^\[Pipeline(?:FullDetailHash|WholeViewHash|LODHash|ExactQueryHash)\].*$", re.M) STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M) def run(dataset: str, recipe: str, workers: int, compiler: str) -> dict: OUT.mkdir(parents=True, exist_ok=True) label = f"{dataset}-{recipe}-w{workers}-{compiler}" log = OUT / f"{label}.log" layout = OUT / f"{label}.layout.json" command = [ "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", str(BINARIES[compiler]), str(DATASETS[dataset]), f"--workers={workers}", "--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline", "--force-generate", "--clip-radius=3", "--max-clip-level=8", "--max-level=8", f"--recipe={recipe}", "--stream-hash", f"--resolved-layout-output={layout}", ] started = time.monotonic() with log.open("w", encoding="utf-8") as stream: result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, check=False, timeout=1740) output = log.read_text(encoding="utf-8", errors="replace") hashes = HASH.findall(output) if result.returncode or len(hashes) != 12: raise SystemExit(f"hash run failed or incomplete: {label}; exit={result.returncode} " f"hashes={len(hashes)} log={log}") row = {"dataset": dataset, "recipe": recipe, "workers": workers, "compiler": compiler, "exit_code": result.returncode, "hashes": hashes, "source_stamp": (STAMP.search(output).group(1) if STAMP.search(output) else None), "elapsed_s": round(time.monotonic() - started, 3), "log": str(log.relative_to(ROOT))} print("[L109HashRun] " + json.dumps(row, sort_keys=True), flush=True) return row def main() -> int: if any(not path.is_file() for path in (*DATASETS.values(), *BINARIES.values())): raise SystemExit("missing L109 hash input") rows = [] recipes = {"owner4": ("surface-world",), "owner2": ("surface-world", "floating-islands")} for dataset, dataset_recipes in recipes.items(): for recipe in dataset_recipes: for workers in (1, 8, 12): for compiler in ("clang", "gcc"): rows.append(run(dataset, recipe, workers, compiler)) (OUT / "runs-partial.json").write_text( json.dumps(rows, indent=2) + "\n", encoding="utf-8") groups = {} for dataset, dataset_recipes in recipes.items(): for recipe in dataset_recipes: key = f"{dataset}/{recipe}" group = [row for row in rows if row["dataset"] == dataset and row["recipe"] == recipe] reference = group[0]["hashes"] groups[key] = {"all_worker_compiler_hashes_match": all( row["hashes"] == reference for row in group), "source_stamps": sorted({row["source_stamp"] for row in group}), "runs": [{"workers": row["workers"], "compiler": row["compiler"], "log": row["log"]} for row in group]} if not groups[key]["all_worker_compiler_hashes_match"]: raise SystemExit(f"determinism mismatch: {key}") if dataset == "owner2": canonical = next(row for row in group if row["workers"] == 12 and row["compiler"] == "clang") recipe_name = "floating-islands" if recipe == "floating-islands" else "surface-world" style = 1 if recipe_name == "surface-world" else 2 baseline = (ROOT / "Standalone/Tools/verify/baselines/owner-hash/canonical" / f"owner-hash-{recipe_name}-style{style}-v217.baseline") baseline.write_text("\n".join(canonical["hashes"]) + "\n", encoding="utf-8") summary = {"runs": rows, "groups": groups, "owner4_all_four_strata": "owner4/surface-world", "owner2_canonical_pins": ["owner2/surface-world", "owner2/floating-islands"]} (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") return 0 if __name__ == "__main__": raise SystemExit(main())