#!/usr/bin/env python3 """Rerun the L90 candidate census at a measured source floor and two brick sizes.""" from __future__ import annotations import argparse import json import re import subprocess import time from pathlib import Path ROOT = Path(__file__).resolve().parents[4] OUT = ROOT / "Saved/Linux/l91/candidate-census" DATA = ROOT / "Standalone/data/owner_world.json" BINARY = ROOT / "Saved/Linux/l91/bin/VoxelForgeCoreHeadless-l91-clang19" STRATA = { "S0": {"camera": (8.0, 16.0, -372.2626953125), "near": 1408}, "FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216}, "TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344}, "S3": {"camera": (-20.0625, -5.068359375, -900.353515625), "near": 1344}, } FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") BUILD = re.compile(r"^\[L90PassageCandidateBuild\] (.*)$", re.M) CENSUS = re.compile(r"^\[L90PassageCandidateTile\] (.*)$", re.M) NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M) HISTS = ("carve_hist", "tread_hist", "clearance_hist", "mouth_column_hist") SEMANTICS = tuple(name.removesuffix("_hist") for name in HISTS) def parse_fields(line: str) -> dict[str, str]: return dict(FIELDS.findall(line)) def histogram(value: str) -> list[int]: items = [int(item) for item in value.split(",")] if len(items) != 17: raise ValueError(f"expected 17 histogram bins, got {len(items)}") return items def percentile(values: list[int], p: float, active: bool = False) -> int: start = 1 if active else 0 population = sum(values[start:]) target = max(1, int(p * population + 0.999999999)) seen = 0 for size in range(start, len(values)): seen += values[size] if seen >= target: return size return 16 def run(stratum: str, floor: float, brick: float) -> dict: OUT.mkdir(parents=True, exist_ok=True) spec = STRATA[stratum] reach = floor * 0.25 + 0.24 camera = ",".join(f"{value:.12f}" for value in spec["camera"]) + ",0,0" log = OUT / f"{stratum.lower()}-c{floor:g}-b{brick:g}m-w8.log" command = [ "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "env", "VOXELFORGE_L90_PASSAGE_CANDIDATE_CENSUS=1", "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", f"VOXELFORGE_L91_PASSAGE_SOURCE_FLOOR_VOXELS={floor:g}", f"VOXELFORGE_L90_PASSAGE_REACH_METRES={reach:.6f}", f"VOXELFORGE_L90_PASSAGE_BRICK_METRES={brick:g}", "timeout", "1740s", str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline", "--workers=8", "--clip-radius=3", "--max-clip-level=8", "--max-level=8", f"--camera={camera}", "--profile-lod-work", "--profile-passage-composer", ] started = time.monotonic() try: with log.open("w", encoding="utf-8") as stream: result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, check=False, timeout=1750) except subprocess.TimeoutExpired: result = None content = log.read_text(encoding="utf-8", errors="replace") builds = [parse_fields(match.group(1)) for match in BUILD.finditer(content)] near = NEAR.search(content) rows = [] for match in CENSUS.finditer(content): fields = parse_fields(match.group(1)) if int(fields["lod"]) > 2: continue rows.append(fields) histograms = {name.removesuffix("_hist"): [0] * 17 for name in HISTS} fallback_ns = query_ns = 0 for row in rows: fallback_ns += int(row["fallback_ns"]) query_ns += int(row["query_ns"]) for name in HISTS: values = histogram(row[name]) key = name.removesuffix("_hist") histograms[key] = [a + b for a, b in zip(histograms[key], values)] metrics = {} for name, values in histograms.items(): metrics[name] = { "queries": sum(values), "active_p90": percentile(values, 0.90, active=True), "active_p95": percentile(values, 0.95, active=True), "all_p90": percentile(values, 0.90), "histogram_0_to_15_plus": values, } row = { "stratum": stratum, "floor_vox": floor, "reach_m": round(reach, 6), "brick_m": brick, "workers": 8, "exit_code": result.returncode if result is not None else "timeout", "wall_s": round(time.monotonic() - started, 3), "near_tiles": spec["near"], "near_complete": bool(near and int(near.group(1)) == spec["near"] and int(near.group(2)) == spec["near"]), "lod02_ready_s": float(near.group(3)) if near else None, "census_rows": len(rows), "census_queries": sum(v["queries"] for v in metrics.values()) // 4, "fallback_ns": fallback_ns, "query_ns": query_ns, "fallback_share": fallback_ns / query_ns if query_ns else None, "histograms": metrics, "build": builds[-1] if builds else None, "source_stamp": STAMP.search(content).group(1) if STAMP.search(content) else None, "log": str(log.relative_to(ROOT)), } print("[L91CandidateCensusRun] " + json.dumps(row, sort_keys=True), flush=True) return row def aggregate(rows: list[dict]) -> dict: output = {} for brick in sorted({row["brick_m"] for row in rows}): selected = [row for row in rows if row["brick_m"] == brick] combined = {name: [0] * 17 for name in SEMANTICS} fallback_ns = query_ns = 0 for row in selected: fallback_ns += row["fallback_ns"] query_ns += row["query_ns"] for name in SEMANTICS: values = row["histograms"][name]["histogram_0_to_15_plus"] combined[name] = [a + b for a, b in zip(combined[name], values)] distributions = {name: { "queries": sum(values), "all_p90": percentile(values, 0.90), "active_p90": percentile(values, 0.90, active=True), "active_p95": percentile(values, 0.95, active=True), "histogram_0_to_15_plus": values, } for name, values in combined.items()} weighted_fallback = fallback_ns / query_ns if query_ns else 1.0 stratum_fallback = {row["stratum"]: row["fallback_share"] for row in selected} p90_ok = distributions["carve"]["active_p90"] <= 6 \ and distributions["mouth_column"]["active_p90"] <= 6 fallback_ok = weighted_fallback < 0.10 and all( value is not None and value < 0.10 for value in stratum_fallback.values()) output[str(brick)] = { "strata": [row["stratum"] for row in selected], "weighted_fallback_share": weighted_fallback, "stratum_fallback_share": stratum_fallback, "weighted_semantics": distributions, "p90_pass": p90_ok, "fallback_pass": fallback_ok, "census_pass": p90_ok and fallback_ok and len(selected) == 4 and all(row["near_complete"] and row["exit_code"] == 0 for row in selected), } return output def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--floor", type=float, required=True) parser.add_argument("--bricks", type=float, nargs="+", default=[2.0, 4.0]) parser.add_argument("--strata", nargs="+", choices=tuple(STRATA), default=tuple(STRATA)) args = parser.parse_args() if not BINARY.is_file() or not DATA.is_file(): raise SystemExit(f"missing input: {BINARY} or {DATA}") rows = [run(stratum, args.floor, brick) for brick in args.bricks for stratum in args.strata] payload = {"floor_vox": args.floor, "reach_m": args.floor * 0.25 + 0.24, "runs": rows, "aggregates_by_brick_m": aggregate(rows)} (OUT / "summary.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") return 0 if all(value["census_pass"] for value in payload["aggregates_by_brick_m"].values()) else 1 if __name__ == "__main__": raise SystemExit(main())