VoxelForge/Standalone/Research/WorldAssemblerM3/Linux91/RunL91CandidateCensus.py

#!/usr/bin/env python3
"""Rerun the L90 candidate census at a measured source floor and two brick sizes."""
from __future__ import annotations

import argparse
import json
import re
import subprocess
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l91/candidate-census"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARY = ROOT / "Saved/Linux/l91/bin/VoxelForgeCoreHeadless-l91-clang19"
STRATA = {
    "S0": {"camera": (8.0, 16.0, -372.2626953125), "near": 1408},
    "FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216},
    "TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344},
    "S3": {"camera": (-20.0625, -5.068359375, -900.353515625), "near": 1344},
}
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
BUILD = re.compile(r"^\[L90PassageCandidateBuild\] (.*)$", re.M)
CENSUS = re.compile(r"^\[L90PassageCandidateTile\] (.*)$", re.M)
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
HISTS = ("carve_hist", "tread_hist", "clearance_hist", "mouth_column_hist")
SEMANTICS = tuple(name.removesuffix("_hist") for name in HISTS)


def parse_fields(line: str) -> dict[str, str]:
    return dict(FIELDS.findall(line))


def histogram(value: str) -> list[int]:
    items = [int(item) for item in value.split(",")]
    if len(items) != 17:
        raise ValueError(f"expected 17 histogram bins, got {len(items)}")
    return items


def percentile(values: list[int], p: float, active: bool = False) -> int:
    start = 1 if active else 0
    population = sum(values[start:])
    target = max(1, int(p * population + 0.999999999))
    seen = 0
    for size in range(start, len(values)):
        seen += values[size]
        if seen >= target:
            return size
    return 16


def run(stratum: str, floor: float, brick: float) -> dict:
    OUT.mkdir(parents=True, exist_ok=True)
    spec = STRATA[stratum]
    reach = floor * 0.25 + 0.24
    camera = ",".join(f"{value:.12f}" for value in spec["camera"]) + ",0,0"
    log = OUT / f"{stratum.lower()}-c{floor:g}-b{brick:g}m-w8.log"
    command = [
        "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "env",
        "VOXELFORGE_L90_PASSAGE_CANDIDATE_CENSUS=1",
        "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0",
        f"VOXELFORGE_L91_PASSAGE_SOURCE_FLOOR_VOXELS={floor:g}",
        f"VOXELFORGE_L90_PASSAGE_REACH_METRES={reach:.6f}",
        f"VOXELFORGE_L90_PASSAGE_BRICK_METRES={brick:g}",
        "timeout", "1740s", str(BINARY), str(DATA), "--world-assembler-prototype",
        "--stream-benchmark", "--stream-pipeline", "--workers=8", "--clip-radius=3",
        "--max-clip-level=8", "--max-level=8", f"--camera={camera}",
        "--profile-lod-work", "--profile-passage-composer",
    ]
    started = time.monotonic()
    try:
        with log.open("w", encoding="utf-8") as stream:
            result = subprocess.run(command, cwd=ROOT, stdout=stream,
                                    stderr=subprocess.STDOUT, check=False, timeout=1750)
    except subprocess.TimeoutExpired:
        result = None
    content = log.read_text(encoding="utf-8", errors="replace")
    builds = [parse_fields(match.group(1)) for match in BUILD.finditer(content)]
    near = NEAR.search(content)
    rows = []
    for match in CENSUS.finditer(content):
        fields = parse_fields(match.group(1))
        if int(fields["lod"]) > 2:
            continue
        rows.append(fields)
    histograms = {name.removesuffix("_hist"): [0] * 17 for name in HISTS}
    fallback_ns = query_ns = 0
    for row in rows:
        fallback_ns += int(row["fallback_ns"])
        query_ns += int(row["query_ns"])
        for name in HISTS:
            values = histogram(row[name])
            key = name.removesuffix("_hist")
            histograms[key] = [a + b for a, b in zip(histograms[key], values)]
    metrics = {}
    for name, values in histograms.items():
        metrics[name] = {
            "queries": sum(values),
            "active_p90": percentile(values, 0.90, active=True),
            "active_p95": percentile(values, 0.95, active=True),
            "all_p90": percentile(values, 0.90),
            "histogram_0_to_15_plus": values,
        }
    row = {
        "stratum": stratum, "floor_vox": floor, "reach_m": round(reach, 6),
        "brick_m": brick, "workers": 8,
        "exit_code": result.returncode if result is not None else "timeout",
        "wall_s": round(time.monotonic() - started, 3),
        "near_tiles": spec["near"],
        "near_complete": bool(near and int(near.group(1)) == spec["near"]
                               and int(near.group(2)) == spec["near"]),
        "lod02_ready_s": float(near.group(3)) if near else None,
        "census_rows": len(rows), "census_queries": sum(v["queries"] for v in metrics.values()) // 4,
        "fallback_ns": fallback_ns, "query_ns": query_ns,
        "fallback_share": fallback_ns / query_ns if query_ns else None,
        "histograms": metrics, "build": builds[-1] if builds else None,
        "source_stamp": STAMP.search(content).group(1) if STAMP.search(content) else None,
        "log": str(log.relative_to(ROOT)),
    }
    print("[L91CandidateCensusRun] " + json.dumps(row, sort_keys=True), flush=True)
    return row


def aggregate(rows: list[dict]) -> dict:
    output = {}
    for brick in sorted({row["brick_m"] for row in rows}):
        selected = [row for row in rows if row["brick_m"] == brick]
        combined = {name: [0] * 17 for name in SEMANTICS}
        fallback_ns = query_ns = 0
        for row in selected:
            fallback_ns += row["fallback_ns"]
            query_ns += row["query_ns"]
            for name in SEMANTICS:
                values = row["histograms"][name]["histogram_0_to_15_plus"]
                combined[name] = [a + b for a, b in zip(combined[name], values)]
        distributions = {name: {
            "queries": sum(values),
            "all_p90": percentile(values, 0.90),
            "active_p90": percentile(values, 0.90, active=True),
            "active_p95": percentile(values, 0.95, active=True),
            "histogram_0_to_15_plus": values,
        } for name, values in combined.items()}
        weighted_fallback = fallback_ns / query_ns if query_ns else 1.0
        stratum_fallback = {row["stratum"]: row["fallback_share"] for row in selected}
        p90_ok = distributions["carve"]["active_p90"] <= 6 \
            and distributions["mouth_column"]["active_p90"] <= 6
        fallback_ok = weighted_fallback < 0.10 and all(
            value is not None and value < 0.10 for value in stratum_fallback.values())
        output[str(brick)] = {
            "strata": [row["stratum"] for row in selected],
            "weighted_fallback_share": weighted_fallback,
            "stratum_fallback_share": stratum_fallback,
            "weighted_semantics": distributions,
            "p90_pass": p90_ok, "fallback_pass": fallback_ok,
            "census_pass": p90_ok and fallback_ok and len(selected) == 4
                and all(row["near_complete"] and row["exit_code"] == 0 for row in selected),
        }
    return output


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--floor", type=float, required=True)
    parser.add_argument("--bricks", type=float, nargs="+", default=[2.0, 4.0])
    parser.add_argument("--strata", nargs="+", choices=tuple(STRATA), default=tuple(STRATA))
    args = parser.parse_args()
    if not BINARY.is_file() or not DATA.is_file():
        raise SystemExit(f"missing input: {BINARY} or {DATA}")
    rows = [run(stratum, args.floor, brick)
            for brick in args.bricks for stratum in args.strata]
    payload = {"floor_vox": args.floor, "reach_m": args.floor * 0.25 + 0.24,
               "runs": rows, "aggregates_by_brick_m": aggregate(rows)}
    (OUT / "summary.json").write_text(json.dumps(payload, indent=2) + "\n",
                                       encoding="utf-8")
    return 0 if all(value["census_pass"]
                    for value in payload["aggregates_by_brick_m"].values()) else 1


if __name__ == "__main__":
    raise SystemExit(main())