VoxelForge/Standalone/Research/WorldAssemblerM3/Linux74/RunHashSweepL74.py

#!/usr/bin/env python3
"""Exact full-grid versus whole-tile skip sweep at 20 camera positions per stratum."""
from __future__ import annotations

import json
import re
import statistics
import subprocess
import time
from pathlib import Path


ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l74/logs"
DATA = ROOT / "Standalone/data/owner_world.json"
BIN = ROOT / "Saved/Linux/l74/bin/VoxelForgeCoreHeadless-l74-clang"
CAMERAS = {
    0: (8.0, 16.0, -372.2626953125),
    1: (25.8369140625, 3.19921875, -737.0),
    2: (19.166015625, -14.2001953125, -753.3740234375),
    3: (-20.0625, -5.068359375, -900.353515625),
}
OFFSETS = [(x, y) for y in (0.1, 4.1, 8.1, 12.1)
           for x in (0.1, 4.1, 8.1, 12.1, 16.1)]
HASH = re.compile(
    r"\[PipelineTileHash\] lod=(\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
    r"density=(0x[0-9a-fA-F]+) mesh=(0x[0-9a-fA-F]+) "
    r"payload=(0x[0-9a-fA-F]+) topology=(0x[0-9a-fA-F]+)"
)
GENERATED = re.compile(
    r"\[UniformGridCeiling\] strate=(-?\d+) lod=(\d+) key="
    r"\((-?\d+),(-?\d+),(-?\d+)\)"
)
PIPELINE = re.compile(r"\[StreamPipeline\] (.*)")
TILE_PIPELINE = re.compile(r"\[WorldAssemblerTilePipeline\] (.*)")
PAIR = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")


def read_hashes(path: Path) -> tuple[dict, set[tuple[int, int, int, int]]]:
    hashes = {}
    generated = set()
    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
        match = HASH.search(line)
        if match:
            lod, x, y, z = (int(match.group(index)) for index in range(1, 5))
            hashes[(lod, x, y, z)] = {
                "density": match.group(5).lower(), "mesh": match.group(6).lower(),
                "payload": match.group(7).lower(), "topology": match.group(8).lower(),
            }
        match = GENERATED.search(line)
        if match:
            generated.add(tuple(int(match.group(index)) for index in range(2, 6)))
    return hashes, generated


def parse_summary(path: Path) -> dict:
    content = path.read_text(encoding="utf-8", errors="replace")
    pipeline = PIPELINE.search(content)
    tile_pipeline = TILE_PIPELINE.search(content)
    if pipeline is None or tile_pipeline is None:
        raise SystemExit(f"pipeline summaries missing; see {path}")
    values = dict(PAIR.findall(pipeline.group(1)))
    tile_values = dict(PAIR.findall(tile_pipeline.group(1)))
    tiles = values.get("whole_tiles", "0/0").split("/")
    return {
        "tiles": int(tiles[1]) if len(tiles) == 2 else 0,
        "whole_tile_skips": int(tile_values.get("whole_tile_skips", "0")),
        "classification_worker_ms": float(tile_values.get("classification_worker_ms", "0")),
    }


def run(mode: str, view: int, position: int, offset: tuple[float, float]) -> dict:
    base = CAMERAS[view]
    xyz = (base[0] + offset[0], base[1] + offset[1], base[2])
    camera = ",".join(f"{value:.12f}" for value in xyz) + ",0,0"
    log = OUT / f"sweep-{mode}-s{view}-p{position:02d}.log"
    hash_flag = "--stream-hash-tiles" if mode == "reference" \
        else "--stream-hash-tiles-allow-skips"
    command = [
        "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout",
        "1740s", str(BIN), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
        "--stream-pipeline", "--force-generate", "--workers=8", "--clip-radius=3",
        "--max-clip-level=8", "--max-level=8", f"--camera={camera}", hash_flag,
    ]
    started = time.monotonic()
    with log.open("w", encoding="utf-8") as output:
        result = subprocess.run(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT,
                                check=False, timeout=1740)
    hashes, _ = read_hashes(log)
    if result.returncode != 0 or not hashes:
        raise SystemExit(f"{mode} hash generation failed; see {log}")
    summary = parse_summary(log)
    row = {
        "mode": mode, "strate": view, "position": position, "offset_m": list(offset),
        "camera_m": list(xyz), "exit_code": result.returncode,
        "wall_s": round(time.monotonic() - started, 3), "hash_rows": len(hashes),
        **summary, "log": str(log.relative_to(ROOT)),
    }
    print("[L74HashSweepRun] " + json.dumps(row, sort_keys=True), flush=True)
    return row


def compare(reference: Path, candidate: Path) -> dict:
    expected, _ = read_hashes(reference)
    actual, candidate_generated = read_hashes(candidate)
    missing = expected.keys() - actual.keys()
    extra = actual.keys() - expected.keys()
    mismatches = {field: 0 for field in ("mesh", "payload", "topology", "density_processed")}
    for key in expected.keys() & actual.keys():
        fields = ["mesh", "payload", "topology"]
        if key in candidate_generated:
            fields.append("density")
        for field in fields:
            if expected[key][field] != actual[key][field]:
                mismatches["density_processed" if field == "density" else field] += 1
    return {
        "reference": str(reference.relative_to(ROOT)),
        "candidate": str(candidate.relative_to(ROOT)),
        "reference_tiles": len(expected), "candidate_tiles": len(actual),
        "missing_keys": len(missing), "extra_keys": len(extra),
        "mismatches": mismatches,
        "equal": not missing and not extra and not any(mismatches.values()),
    }


def main() -> int:
    if not DATA.is_file() or not BIN.is_file():
        raise SystemExit("missing owner world or L74 Clang headless binary")
    references, candidates, comparisons = [], [], []
    for view in range(4):
        for position, offset in enumerate(OFFSETS):
            reference = run("reference", view, position, offset)
            candidate = run("skip", view, position, offset)
            comparison = compare(ROOT / reference["log"], ROOT / candidate["log"])
            comparison.update(strate=view, position=position)
            references.append(reference)
            candidates.append(candidate)
            comparisons.append(comparison)
            print("[L74HashSweepCompare] " + json.dumps(comparison, sort_keys=True), flush=True)
            if not comparison["equal"]:
                raise SystemExit(f"whole-tile proof falsified; see {comparison['candidate']}")
    summaries = []
    for view in range(4):
        rows = [row for row in candidates if row["strate"] == view]
        skips = sum(row["whole_tile_skips"] for row in rows)
        tiles = sum(row["tiles"] for row in rows)
        costs = [row["classification_worker_ms"] for row in rows]
        summaries.append({
            "strate": view, "camera_positions": len(rows), "hash_comparisons": len(rows),
            "violations": sum(not row["equal"] for row in comparisons
                              if row["strate"] == view),
            "whole_tile_proof_hits": skips, "tiles_visited": tiles,
            "proof_hit_pct": round(100.0 * skips / max(tiles, 1), 2),
            "classifier_ms_p50": round(statistics.median(costs), 3),
            "classifier_ms_p95": round(sorted(costs)[int(0.95 * (len(costs) - 1))], 3),
        })
    result = {"runs": {"reference": references, "skip": candidates},
              "comparisons": comparisons, "summary": summaries}
    (OUT / "hash-sweep.json").write_text(json.dumps(result, indent=2) + "\n",
                                         encoding="utf-8")
    for row in summaries:
        print("[L74HashSweepSummary] " + json.dumps(row, sort_keys=True), flush=True)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())