skip: sound whole-tile proofs for legacy FI/TN/VS fields (L74); FI -30%, TN -22%
The compiled-analytic stratum interval now certifies a legacy tile uniform only when it stays beyond the detail envelope (and smooth-min slack for solid) with no intersecting primitive, edit, native field, composer candidate, passage or active transition; everything else fails closed. The stream layer uses the proof for whole-tile skips; legacy sub-tile skips stay off (two block proofs failed the falsifier in TN). Falsifier: full-grid reference vs skip-enabled run, identical mesh/payload/topology hashes on every tile of all four owner views, Clang and GCC, W1/W8/W12; 20-camera sweep per strate, 0 violations; a too-tight control changed 3/184/233/238 tiles and was caught. Whole view (8 workers): SW 2.61->2.48, FI 7.18->5.05, TN 5.97->4.63, VS 2.70->2.40 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed
1 parent
bd1b2a8f1d
commit
c67b2006aa
15 files changed
+1024
-21
No files matched your search
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare full-grid and proof-skip mesh hashes for all four owner arrivals."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l74/logs"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BINS = {
|
||||
"clang": ROOT / "Saved/Linux/l74/bin/VoxelForgeCoreHeadless-l74-clang",
|
||||
"gcc": ROOT / "Saved/Linux/l74/bin-gcc/VoxelForgeCoreHeadless-l74-gcc",
|
||||
}
|
||||
CAMERAS = {
|
||||
0: (8.0, 16.0, -372.2626953125),
|
||||
1: (25.8369140625, 3.19921875, -737.0),
|
||||
2: (19.166015625, -14.2001953125, -753.3740234375),
|
||||
3: (-20.0625, -5.068359375, -900.353515625),
|
||||
}
|
||||
HASH_ROW = re.compile(
|
||||
r"\[PipelineTileHash\] lod=(\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
|
||||
r"density=(0x[0-9a-fA-F]+) mesh=(0x[0-9a-fA-F]+) "
|
||||
r"payload=(0x[0-9a-fA-F]+) topology=(0x[0-9a-fA-F]+)"
|
||||
)
|
||||
CEILING_ROW = re.compile(
|
||||
r"\[UniformGridCeiling\] strate=(-?\d+) lod=(\d+) key=\((-?\d+),(-?\d+),(-?\d+)\)"
|
||||
)
|
||||
|
||||
|
||||
def read_hashes(path: Path) -> tuple[dict, set[tuple[int, int, int, int]]]:
|
||||
hashes = {}
|
||||
generated_density = set()
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
row = HASH_ROW.search(line)
|
||||
if row:
|
||||
lod, x, y, z = (int(row.group(i)) for i in range(1, 5))
|
||||
hashes[(lod, x, y, z)] = {
|
||||
"density": row.group(5).lower(), "mesh": row.group(6).lower(),
|
||||
"payload": row.group(7).lower(), "topology": row.group(8).lower(),
|
||||
}
|
||||
ceiling = CEILING_ROW.search(line)
|
||||
if ceiling:
|
||||
generated_density.add(tuple(int(ceiling.group(i)) for i in range(2, 6)))
|
||||
return hashes, generated_density
|
||||
|
||||
|
||||
def run(compiler: str, view: int, workers: int, allow_skips: bool,
|
||||
negative_control: bool = False) -> Path:
|
||||
camera = ",".join(f"{value:.12f}" for value in CAMERAS[view]) + ",0,0"
|
||||
mode = "negative" if negative_control else "skip" if allow_skips else "reference"
|
||||
log = OUT / f"{mode}-{compiler}-view{view}-w{workers}.log"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||
"timeout", "1740s", str(BINS[compiler]), str(DATA),
|
||||
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
|
||||
"--force-generate", f"--workers={workers}", "--clip-radius=3",
|
||||
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
|
||||
"--stream-hash-tiles-allow-skips" if allow_skips or negative_control
|
||||
else "--stream-hash-tiles",
|
||||
]
|
||||
import os
|
||||
environment = os.environ.copy()
|
||||
if negative_control:
|
||||
environment["VOXELFORGE_L74_TIGHT_PROOF_CONTROL"] = "1"
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as output:
|
||||
result = subprocess.run(command, cwd=ROOT, env=environment, stdout=output,
|
||||
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||
hashes, _ = read_hashes(log)
|
||||
print(json.dumps({"compiler": compiler, "view": view, "workers": workers,
|
||||
"mode": mode, "exit_code": result.returncode,
|
||||
"hash_rows": len(hashes), "wall_s": round(time.monotonic()-started, 3),
|
||||
"log": str(log.relative_to(ROOT))}, sort_keys=True), flush=True)
|
||||
if result.returncode != 0 or not hashes:
|
||||
raise SystemExit(f"hash generation failed; see {log}")
|
||||
return log
|
||||
|
||||
|
||||
def compare(reference: Path, candidate: Path) -> dict:
|
||||
reference_hashes, _ = read_hashes(reference)
|
||||
candidate_hashes, candidate_generated = read_hashes(candidate)
|
||||
missing = sorted(reference_hashes.keys() - candidate_hashes.keys())
|
||||
extra = sorted(candidate_hashes.keys() - reference_hashes.keys())
|
||||
mismatch_counts = {field: 0 for field in ("mesh", "payload", "topology", "density_processed")}
|
||||
mismatch_examples = []
|
||||
for key in sorted(reference_hashes.keys() & candidate_hashes.keys()):
|
||||
a, b = reference_hashes[key], candidate_hashes[key]
|
||||
fields = ["mesh", "payload", "topology"]
|
||||
if key in candidate_generated:
|
||||
fields.append("density")
|
||||
for field in fields:
|
||||
if a[field] != b[field]:
|
||||
label = "density_processed" if field == "density" else field
|
||||
mismatch_counts[label] += 1
|
||||
if len(mismatch_examples) < 8:
|
||||
mismatch_examples.append({"key": key, "field": field,
|
||||
"reference": a[field], "candidate": b[field]})
|
||||
return {"reference": str(reference.relative_to(ROOT)),
|
||||
"candidate": str(candidate.relative_to(ROOT)),
|
||||
"reference_tiles": len(reference_hashes), "candidate_tiles": len(candidate_hashes),
|
||||
"missing_keys": missing[:8], "missing_key_count": len(missing),
|
||||
"extra_keys": extra[:8], "extra_key_count": len(extra),
|
||||
"mismatch_counts": mismatch_counts, "mismatch_examples": mismatch_examples,
|
||||
"mesh_topology_equal": not missing and not extra
|
||||
and mismatch_counts["mesh"] == 0 and mismatch_counts["topology"] == 0}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--phase", choices=("matrix", "workers", "negative"), required=True)
|
||||
args = parser.parse_args()
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
missing = [str(path) for path in [DATA, *BINS.values()] if not path.is_file()]
|
||||
if missing:
|
||||
raise SystemExit("missing input: " + ", ".join(missing))
|
||||
comparisons = []
|
||||
if args.phase == "matrix":
|
||||
references = {}
|
||||
skip_eight = {}
|
||||
for compiler in ("clang", "gcc"):
|
||||
for view in range(4):
|
||||
reference = run(compiler, view, 8, False)
|
||||
skip = run(compiler, view, 8, True)
|
||||
references[(compiler, view)] = reference
|
||||
skip_eight[(compiler, view)] = skip
|
||||
comparisons.append(compare(reference, skip))
|
||||
for view in range(4):
|
||||
comparisons.append(compare(references[("clang", view)],
|
||||
references[("gcc", view)]))
|
||||
comparisons.append(compare(skip_eight[("clang", view)],
|
||||
skip_eight[("gcc", view)]))
|
||||
(OUT / "falsifier-matrix.json").write_text(json.dumps(comparisons, indent=2)+"\n",
|
||||
encoding="utf-8")
|
||||
elif args.phase == "workers":
|
||||
for compiler in ("clang", "gcc"):
|
||||
for view in range(4):
|
||||
reference = OUT / f"reference-{compiler}-view{view}-w8.log"
|
||||
if not reference.is_file():
|
||||
raise SystemExit(f"missing reference from matrix phase: {reference}")
|
||||
baseline = OUT / f"skip-{compiler}-view{view}-w8.log"
|
||||
for workers in (1, 12):
|
||||
candidate = run(compiler, view, workers, True)
|
||||
comparisons.append(compare(reference, candidate))
|
||||
comparisons.append(compare(baseline, candidate))
|
||||
(OUT / "falsifier-workers.json").write_text(json.dumps(comparisons, indent=2)+"\n",
|
||||
encoding="utf-8")
|
||||
else:
|
||||
for view in range(4):
|
||||
reference = OUT / f"reference-clang-view{view}-w8.log"
|
||||
candidate = run("clang", view, 8, True, negative_control=True)
|
||||
comparisons.append(compare(reference, candidate))
|
||||
(OUT / "falsifier-negative-control.json").write_text(
|
||||
json.dumps(comparisons, indent=2)+"\n", encoding="utf-8")
|
||||
caught = any(not row["mesh_topology_equal"] for row in comparisons)
|
||||
print(f"[L74NegativeControlResult] caught={int(caught)}", flush=True)
|
||||
if not caught:
|
||||
return 2
|
||||
return 0
|
||||
failures = [row for row in comparisons if not row["mesh_topology_equal"]]
|
||||
print(f"[L74Falsifier] phase={args.phase} comparisons={len(comparisons)} "
|
||||
f"mesh_topology_failures={len(failures)}", flush=True)
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,79 @@
|
||||
# Linux 74 — legacy uniform tile proofs
|
||||
|
||||
## Ceiling
|
||||
|
||||
Full-grid signs were counted per camera observation. “Visited / sampled” separates tiles with no captured density grid; those are not counted as uniform. Block columns are the fraction of captured samples belonging to sign-uniform, disjoint cubes. Partial edge cubes are included. Repeated tile keys in different camera contexts are separate observations. Strate labels are SW=SurfaceWorld, FI=FloatingIslands, TN=TunnelNetwork, VS=VerticalShafts. Unlisted stratum/LOD pairs had no observations.
|
||||
|
||||
| Strate | LOD | Visited / sampled | Uniform tiles / sampled (%); % visited | Samples in uniform 8³ | Samples in uniform 4³ |
|
||||
|---|---:|---:|---:|---:|---:|
|
||||
| SW | 0 | 512 / 147 | 35 / 147 (23.81%); 6.84% | 79.74% | 90.77% |
|
||||
| SW | 1 | 448 / 448 | 317 / 448 (70.76%); 70.76% | 94.33% | 97.36% |
|
||||
| SW | 2 | 448 / 448 | 391 / 448 (87.28%); 87.28% | 92.78% | 97.01% |
|
||||
| SW | 3 | 320 / 320 | 270 / 320 (84.38%); 84.38% | 93.65% | 96.76% |
|
||||
| SW | 4 | 432 / 432 | 48 / 432 (11.11%); 11.11% | 70.51% | 85.37% |
|
||||
| SW | 5 | 288 / 288 | 0 / 288 (0%); 0% | 75.44% | 82.46% |
|
||||
| SW | 6 | 192 / 192 | 0 / 192 (0%); 0% | 42.11% | 63.16% |
|
||||
| FI | 0 | 512 / 287 | 132 / 287 (45.99%); 25.78% | 89.41% | 95.24% |
|
||||
| FI | 1 | 432 / 432 | 339 / 432 (78.47%); 78.47% | 96.73% | 98.55% |
|
||||
| FI | 2 | 272 / 272 | 237 / 272 (87.13%); 87.13% | 95.99% | 98.35% |
|
||||
| FI | 3 | 384 / 384 | 243 / 384 (63.28%); 63.28% | 90.33% | 95.70% |
|
||||
| FI | 4 | 192 / 192 | 56 / 192 (29.17%); 29.17% | 82.44% | 92.62% |
|
||||
| FI | 5 | 144 / 144 | 0 / 144 (0%); 0% | 61.03% | 84.63% |
|
||||
| FI | 7 | 192 / 192 | 0 / 192 (0%); 0% | 0% | 53.04% |
|
||||
| TN | 0 | 576 / 266 | 146 / 266 (54.89%); 25.35% | 92.07% | 96.07% |
|
||||
| TN | 1 | 640 / 640 | 524 / 640 (81.88%); 81.88% | 97.49% | 98.88% |
|
||||
| TN | 2 | 416 / 416 | 250 / 416 (60.10%); 60.10% | 92.03% | 97.15% |
|
||||
| TN | 3 | 240 / 240 | 60 / 240 (25.00%); 25.00% | 73.52% | 88.85% |
|
||||
| TN | 4 | 144 / 144 | 36 / 144 (25.00%); 25.00% | 81.70% | 88.85% |
|
||||
| TN | 6 | 192 / 192 | 0 / 192 (0%); 0% | 25.98% | 57.46% |
|
||||
| VS | 0 | 448 / 226 | 117 / 226 (51.77%); 26.12% | 90.25% | 95.42% |
|
||||
| VS | 1 | 272 / 272 | 218 / 272 (80.15%); 80.15% | 97.05% | 98.66% |
|
||||
| VS | 2 | 272 / 272 | 171 / 272 (62.87%); 62.87% | 92.39% | 97.29% |
|
||||
| VS | 3 | 160 / 160 | 20 / 160 (12.50%); 12.50% | 67.72% | 84.82% |
|
||||
| VS | 4 | 96 / 96 | 0 / 96 (0%); 0% | 40.22% | 88.00% |
|
||||
| VS | 5 | 144 / 144 | 0 / 144 (0%); 0% | 29.12% | 56.92% |
|
||||
|
||||
The 8³/4³ ceiling is high, but legacy sub-tile proofs did not survive the falsifier. The safe implementation below uses whole-tile skips only for legacy fields.
|
||||
|
||||
## Method and falsifier
|
||||
|
||||
`ShapeWorldInternal.h` now marks a legacy tile skippable only when the compiled analytic stratum interval stays beyond the detail envelope (and smooth-min slack for solid), with no intersecting primitive, edit, native field, composer candidate, passage, or active boundary transition. Unbounded or mixed cases fail closed. The shared stream layer uses this proof for FI/TN/VS whole tiles. Legacy block early-outs remain off; native block early-outs remain native-only.
|
||||
|
||||
The full-grid `--stream-hash` reference and `--stream-hash-tiles-allow-skips` candidate matched all per-tile mesh, payload, and topology hashes for all four views on Clang and GCC: 16 matrix comparisons, no missing/extra tile keys, zero geometry mismatches. Processed density hashes also matched. Tile counts were SW 2,850, FI 2,249, TN 2,233, VS 2,329. The W1/W8/W12 matrix added 32 exact comparisons with zero mismatches.
|
||||
|
||||
The deliberately tight control omits the primitive/composer and detail envelope. It was caught in every view: changed mesh tiles SW 3, FI 184, TN 233, VS 238. The initial detail-only control was not caught; the control was strengthened before accepting the proof.
|
||||
|
||||
### 20-position sweep
|
||||
|
||||
Each stratum had 20 passing camera positions. `classifier ms` is aggregate tile-worker time per view (p50 / p95). Every reference/skip pair matched exact geometry hashes; violations: 0.
|
||||
|
||||
| Strate | Proof hits / tile visits | Hit rate | Classifier ms p50 / p95 |
|
||||
|---|---:|---:|---:|
|
||||
| SW | 13,594 / 57,000 | 23.85% | 566.399 / 587.473 |
|
||||
| FI | 9,786 / 44,980 | 21.76% | 3.728 / 4.086 |
|
||||
| TN | 11,472 / 44,660 | 25.69% | 2.004 / 2.222 |
|
||||
| VS | 10,328 / 46,580 | 22.17% | 0.875 / 0.919 |
|
||||
|
||||
## Whole-view timing
|
||||
|
||||
Three interleaved clean pairs per view, eight workers. Values are median whole-view readiness seconds, before L73 / after L74.
|
||||
|
||||
| Strate | Before | After | Change |
|
||||
|---|---:|---:|---:|
|
||||
| SW | 2.613 | 2.479 | -5.13% |
|
||||
| FI | 7.178 | 5.052 | -29.62% |
|
||||
| TN | 5.970 | 4.633 | -22.40% |
|
||||
| VS | 2.702 | 2.397 | -11.29% |
|
||||
|
||||
`verify.py fast` passed in 277.0 s (269 passes, five diagnostics); owner passage, transition, and worker gates stayed green. All benchmark and verification processes were wrapped with `taskset -c 0-11 prlimit --as=6442450944`. No Unreal build was needed because root `Source/` did not change.
|
||||
|
||||
## Identity and failed runs
|
||||
|
||||
- Branch / HEAD: `assembler/world` / `bd1b2a8f1d6c97dfabd90b1ee15aece9fcff0aa3`.
|
||||
- Owner manifest: `bcc0d0d33ea39435`; resolved layout: `resolved-layout-v3-fnv64:0589300eb07dfe06`.
|
||||
- Owner recipe evaluator: 4; resolved layout schema: 3; `WorldAssemblyEvaluatorVersion`: 213. No mesh or topology version bump: all generated geometry hashes match the full-grid reference.
|
||||
- Source stamp: `93cd38dc2fb4a6a22b86d8c690ec6074d6eb070898e109f3bc2cb2490476d464`. Clang binary SHA256: `975d65c32ce26573aad3b0ede6a0985dfa2851a516d731842cd0eac25bdccc8b`; GCC: `ad2a936ad310e1e1c28b37b71d9d355f6fb30e9f1bf99ae0b80985b0cd713408`.
|
||||
- The corrected legacy full-grid path changes only raw density hash rows versus the L70 FI fixture (whole view and LOD2); mesh, payload, and topology rows remain equal. The L70 fixture is preserved; fast verification now pins the L74 full-grid signature separately.
|
||||
- Two exploratory legacy block proofs failed in one TN tile at sample `(-80,-20,-3218)` (exact density `+0.063384`); legacy sub-tile skips were disabled. A broad negative camera offset failed the passage gate before generation, so the final sweep stayed within passing camera positions. The first fast run detected the density-signature drift above; after adding the separate L74 full-grid fixture, fast passed.
|
||||
|
||||
No commit, push, or stash was made. Exploratory and final logs are in `Saved/Linux/l74/logs/`.
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit certified legacy density blocks across 20 deterministic camera positions per stratum."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l74/logs"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BIN = ROOT / "Saved/Linux/l74/bin/VoxelForgeCoreHeadless-l74-clang"
|
||||
CAMERAS = {
|
||||
0: (8.0, 16.0, -372.2626953125),
|
||||
1: (25.8369140625, 3.19921875, -737.0),
|
||||
2: (19.166015625, -14.2001953125, -753.3740234375),
|
||||
3: (-20.0625, -5.068359375, -900.353515625),
|
||||
}
|
||||
OFFSETS = [(x, y) for y in (-3, -1, 1, 3) for x in (-4, -2, 0, 2, 4)]
|
||||
AUDIT = re.compile(
|
||||
r"\[UniformBlockProofAudit\] checked_blocks=(\d+) checked_samples=(\d+) "
|
||||
r"fallback_blocks=(\d+) unknown_blocks=(\d+) violations=(\d+)"
|
||||
)
|
||||
TILE = re.compile(
|
||||
r"\[WorldAssemblerTilePipeline\] .*?solid_boxes=(\d+) air_boxes=(\d+) "
|
||||
r"mixed_boxes=(\d+) unknown_boxes=(\d+) whole_tile_skips=(\d+).*?"
|
||||
r"classification_worker_ms=([0-9.]+)"
|
||||
)
|
||||
VIOLATION = re.compile(r"\[UniformBlockProofViolation\]")
|
||||
|
||||
|
||||
def run(view: int, position: int, offset: tuple[int, int]) -> dict:
|
||||
base = CAMERAS[view]
|
||||
camera = ",".join(f"{value:.12f}" for value in
|
||||
(base[0] + offset[0], base[1] + offset[1], base[2])) + ",0,0"
|
||||
log = OUT / f"audit-s{view}-p{position:02d}.log"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout",
|
||||
"1740s", str(BIN), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||
"--stream-pipeline", "--force-generate", "--workers=8", "--clip-radius=3",
|
||||
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
|
||||
"--audit-uniform-block-proofs", "--profile-lod-work",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as output:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT,
|
||||
check=False, timeout=1740)
|
||||
content = log.read_text(encoding="utf-8", errors="replace")
|
||||
audit = AUDIT.search(content)
|
||||
tile = TILE.search(content)
|
||||
if result.returncode != 0 or audit is None or tile is None:
|
||||
raise SystemExit(f"audit run failed or summary is missing; see {log}")
|
||||
checked, samples, fallback, unknown, violations = map(int, audit.groups())
|
||||
solid, air, mixed, tile_unknown, tile_skips = map(int, tile.groups()[:5])
|
||||
classifier_ms = float(tile.group(6))
|
||||
violation_lines = len(VIOLATION.findall(content))
|
||||
row = {
|
||||
"strate": view, "position": position, "offset_m": list(offset),
|
||||
"exit_code": result.returncode, "wall_s": round(time.monotonic() - started, 3),
|
||||
"checked_blocks": checked, "checked_samples": samples,
|
||||
"fallback_blocks": fallback, "unknown_blocks": unknown,
|
||||
"violations": violations + violation_lines,
|
||||
"block_proof_hit_pct": round(100.0 * checked / max(checked + fallback, 1), 2),
|
||||
"tile_solid_boxes": solid, "tile_air_boxes": air, "tile_mixed_boxes": mixed,
|
||||
"tile_unknown_boxes": tile_unknown, "tile_whole_skips": tile_skips,
|
||||
"classification_worker_ms": classifier_ms,
|
||||
"log": str(log.relative_to(ROOT)),
|
||||
}
|
||||
print("[L74AuditSweepRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
if row["violations"] != 0:
|
||||
raise SystemExit(f"uniform block proof violation; see {log}")
|
||||
return row
|
||||
|
||||
|
||||
def main() -> int:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
if not DATA.is_file() or not BIN.is_file():
|
||||
raise SystemExit("missing owner world or L74 Clang headless binary")
|
||||
rows = [run(view, index, offset) for view in range(4)
|
||||
for index, offset in enumerate(OFFSETS)]
|
||||
summary = []
|
||||
for view in range(4):
|
||||
group = [row for row in rows if row["strate"] == view]
|
||||
checked = sum(row["checked_blocks"] for row in group)
|
||||
fallback = sum(row["fallback_blocks"] for row in group)
|
||||
costs = [row["classification_worker_ms"] for row in group]
|
||||
summary.append({
|
||||
"strate": view, "positions": len(group), "violations": sum(
|
||||
row["violations"] for row in group), "checked_blocks": checked,
|
||||
"fallback_blocks": fallback, "proof_hit_pct": round(
|
||||
100.0 * checked / max(checked + fallback, 1), 2),
|
||||
"classifier_ms_p50": round(statistics.median(costs), 3),
|
||||
"classifier_ms_p95": round(sorted(costs)[int(0.95 * (len(costs) - 1))], 3),
|
||||
})
|
||||
result = {"runs": rows, "summary": summary}
|
||||
(OUT / "audit-sweep.json").write_text(json.dumps(result, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
for row in summary:
|
||||
print("[L74AuditSweepSummary] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect full-grid sign-uniformity ceilings for the four owner arrivals."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l74/logs"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BIN = ROOT / "Saved/Linux/l74/bin/VoxelForgeCoreHeadless-l74-clang"
|
||||
CAMERAS = {
|
||||
0: (8.0, 16.0, -372.2626953125),
|
||||
1: (25.8369140625, 3.19921875, -737.0),
|
||||
2: (19.166015625, -14.2001953125, -753.3740234375),
|
||||
3: (-20.0625, -5.068359375, -900.353515625),
|
||||
}
|
||||
ROW = re.compile(r"\[UniformGridCeiling\] (.*)")
|
||||
PAIR = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
|
||||
|
||||
def parse_log(path: Path, view: int) -> list[dict]:
|
||||
rows = []
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
match = ROW.search(line)
|
||||
if not match:
|
||||
continue
|
||||
row = dict(PAIR.findall(match.group(1)))
|
||||
key = tuple(int(value) for value in row["key"].strip("()").split(","))
|
||||
row.update(view=view, lod=int(row["lod"]), strate=int(row["strate"]),
|
||||
key=key, samples=int(row["samples"]), uniform=int(row["uniform"]),
|
||||
air=int(row["air"]), blocks8=int(row["blocks8"]),
|
||||
samples8=int(row["samples8"]), blocks4=int(row["blocks4"]),
|
||||
samples4=int(row["samples4"]))
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def run(view: int) -> list[dict]:
|
||||
camera = ",".join(f"{value:.12f}" for value in CAMERAS[view]) + ",0,0"
|
||||
log = OUT / f"ceiling-view{view}.log"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||
"timeout", "1740s", str(BIN), str(DATA), "--world-assembler-prototype",
|
||||
"--stream-benchmark", "--stream-pipeline", "--stream-hash-tiles",
|
||||
"--force-generate", "--workers=8", "--clip-radius=3", "--max-clip-level=8",
|
||||
"--max-level=8", f"--camera={camera}",
|
||||
]
|
||||
start = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as output:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT,
|
||||
check=False, timeout=1740)
|
||||
rows = parse_log(log, view)
|
||||
print(json.dumps({"view": view, "exit_code": result.returncode,
|
||||
"wall_s": round(time.monotonic() - start, 3),
|
||||
"tiles": len(rows), "log": str(log.relative_to(ROOT))}, sort_keys=True),
|
||||
flush=True)
|
||||
if result.returncode != 0 or not rows:
|
||||
raise SystemExit(f"ceiling view {view} failed or emitted no rows; see {log}")
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not DATA.is_file() or not BIN.is_file():
|
||||
raise SystemExit("missing owner world or L74 headless binary")
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
if sys.argv[1:] == ["--analyze-only"]:
|
||||
rows_path = OUT / "ceiling-rows.json"
|
||||
if not rows_path.is_file():
|
||||
raise SystemExit("no recorded ceiling rows to analyze")
|
||||
all_rows = json.loads(rows_path.read_text(encoding="utf-8"))
|
||||
elif len(sys.argv) == 1:
|
||||
all_rows = [row for view in range(4) for row in run(view)]
|
||||
else:
|
||||
raise SystemExit("usage: RunCeilingL74.py [--analyze-only]")
|
||||
unique = {}
|
||||
for row in all_rows:
|
||||
row["key"] = tuple(row["key"])
|
||||
# A tile's compiled field can depend on whether the whole-view build contains it
|
||||
# in its fine region, so retain each of the four camera-context observations.
|
||||
identity = (row["view"], row["strate"], row["lod"], row["key"])
|
||||
previous = unique.get(identity)
|
||||
if previous is None or (previous["samples"] == 0 and row["samples"] > 0):
|
||||
unique[identity] = row
|
||||
elif previous["samples"] > 0 and row["samples"] > 0:
|
||||
signature = ("samples", "uniform", "air", "samples8", "samples4")
|
||||
if any(previous[field] != row[field] for field in signature):
|
||||
raise SystemExit(f"conflicting full-grid ceiling rows for tile {identity}")
|
||||
groups: dict[tuple[int, int], list[dict]] = defaultdict(list)
|
||||
for row in unique.values():
|
||||
groups[(row["strate"], row["lod"])].append(row)
|
||||
summary = []
|
||||
for (strate, lod), rows in sorted(groups.items()):
|
||||
tiles = len(rows)
|
||||
sampled_rows = [row for row in rows if row["samples"] > 0]
|
||||
sampled_tiles = len(sampled_rows)
|
||||
samples = sum(row["samples"] for row in sampled_rows)
|
||||
uniform_tiles = sum(row["uniform"] for row in sampled_rows)
|
||||
uniform_samples = sum(row["samples"] for row in rows if row["uniform"])
|
||||
row = {
|
||||
"strate": strate, "lod": lod, "tiles_visited": tiles,
|
||||
"sampled_tiles": sampled_tiles, "grid_samples": samples,
|
||||
"uniform_tiles": uniform_tiles,
|
||||
"uniform_tile_pct": round(100 * uniform_tiles / sampled_tiles, 2)
|
||||
if sampled_tiles else 0,
|
||||
"uniform_tile_pct_of_visited": round(100 * uniform_tiles / tiles, 2)
|
||||
if tiles else 0,
|
||||
"uniform_air_tiles": sum(row["uniform"] and row["air"] for row in sampled_rows),
|
||||
"uniform_solid_tiles": sum(row["uniform"] and not row["air"]
|
||||
for row in sampled_rows),
|
||||
"samples_in_uniform_tiles_pct": round(100 * uniform_samples / samples, 2)
|
||||
if samples else 0,
|
||||
"uniform_8cube_samples_pct": round(100 * sum(row["samples8"] for row in rows)
|
||||
/ samples, 2) if samples else 0,
|
||||
"uniform_4cube_samples_pct": round(100 * sum(row["samples4"] for row in rows)
|
||||
/ samples, 2) if samples else 0,
|
||||
}
|
||||
summary.append(row)
|
||||
print("[L74Ceiling] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
(OUT / "ceiling-rows.json").write_text(json.dumps(all_rows, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
(OUT / "ceiling-summary.json").write_text(json.dumps(summary, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exact full-grid versus whole-tile skip sweep at 20 camera positions per stratum."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l74/logs"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BIN = ROOT / "Saved/Linux/l74/bin/VoxelForgeCoreHeadless-l74-clang"
|
||||
CAMERAS = {
|
||||
0: (8.0, 16.0, -372.2626953125),
|
||||
1: (25.8369140625, 3.19921875, -737.0),
|
||||
2: (19.166015625, -14.2001953125, -753.3740234375),
|
||||
3: (-20.0625, -5.068359375, -900.353515625),
|
||||
}
|
||||
OFFSETS = [(x, y) for y in (0.1, 4.1, 8.1, 12.1)
|
||||
for x in (0.1, 4.1, 8.1, 12.1, 16.1)]
|
||||
HASH = re.compile(
|
||||
r"\[PipelineTileHash\] lod=(\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
|
||||
r"density=(0x[0-9a-fA-F]+) mesh=(0x[0-9a-fA-F]+) "
|
||||
r"payload=(0x[0-9a-fA-F]+) topology=(0x[0-9a-fA-F]+)"
|
||||
)
|
||||
GENERATED = re.compile(
|
||||
r"\[UniformGridCeiling\] strate=(-?\d+) lod=(\d+) key="
|
||||
r"\((-?\d+),(-?\d+),(-?\d+)\)"
|
||||
)
|
||||
PIPELINE = re.compile(r"\[StreamPipeline\] (.*)")
|
||||
TILE_PIPELINE = re.compile(r"\[WorldAssemblerTilePipeline\] (.*)")
|
||||
PAIR = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
|
||||
|
||||
def read_hashes(path: Path) -> tuple[dict, set[tuple[int, int, int, int]]]:
|
||||
hashes = {}
|
||||
generated = set()
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
match = HASH.search(line)
|
||||
if match:
|
||||
lod, x, y, z = (int(match.group(index)) for index in range(1, 5))
|
||||
hashes[(lod, x, y, z)] = {
|
||||
"density": match.group(5).lower(), "mesh": match.group(6).lower(),
|
||||
"payload": match.group(7).lower(), "topology": match.group(8).lower(),
|
||||
}
|
||||
match = GENERATED.search(line)
|
||||
if match:
|
||||
generated.add(tuple(int(match.group(index)) for index in range(2, 6)))
|
||||
return hashes, generated
|
||||
|
||||
|
||||
def parse_summary(path: Path) -> dict:
|
||||
content = path.read_text(encoding="utf-8", errors="replace")
|
||||
pipeline = PIPELINE.search(content)
|
||||
tile_pipeline = TILE_PIPELINE.search(content)
|
||||
if pipeline is None or tile_pipeline is None:
|
||||
raise SystemExit(f"pipeline summaries missing; see {path}")
|
||||
values = dict(PAIR.findall(pipeline.group(1)))
|
||||
tile_values = dict(PAIR.findall(tile_pipeline.group(1)))
|
||||
tiles = values.get("whole_tiles", "0/0").split("/")
|
||||
return {
|
||||
"tiles": int(tiles[1]) if len(tiles) == 2 else 0,
|
||||
"whole_tile_skips": int(tile_values.get("whole_tile_skips", "0")),
|
||||
"classification_worker_ms": float(tile_values.get("classification_worker_ms", "0")),
|
||||
}
|
||||
|
||||
|
||||
def run(mode: str, view: int, position: int, offset: tuple[float, float]) -> dict:
|
||||
base = CAMERAS[view]
|
||||
xyz = (base[0] + offset[0], base[1] + offset[1], base[2])
|
||||
camera = ",".join(f"{value:.12f}" for value in xyz) + ",0,0"
|
||||
log = OUT / f"sweep-{mode}-s{view}-p{position:02d}.log"
|
||||
hash_flag = "--stream-hash-tiles" if mode == "reference" \
|
||||
else "--stream-hash-tiles-allow-skips"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout",
|
||||
"1740s", str(BIN), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||
"--stream-pipeline", "--force-generate", "--workers=8", "--clip-radius=3",
|
||||
"--max-clip-level=8", "--max-level=8", f"--camera={camera}", hash_flag,
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as output:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT,
|
||||
check=False, timeout=1740)
|
||||
hashes, _ = read_hashes(log)
|
||||
if result.returncode != 0 or not hashes:
|
||||
raise SystemExit(f"{mode} hash generation failed; see {log}")
|
||||
summary = parse_summary(log)
|
||||
row = {
|
||||
"mode": mode, "strate": view, "position": position, "offset_m": list(offset),
|
||||
"camera_m": list(xyz), "exit_code": result.returncode,
|
||||
"wall_s": round(time.monotonic() - started, 3), "hash_rows": len(hashes),
|
||||
**summary, "log": str(log.relative_to(ROOT)),
|
||||
}
|
||||
print("[L74HashSweepRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def compare(reference: Path, candidate: Path) -> dict:
|
||||
expected, _ = read_hashes(reference)
|
||||
actual, candidate_generated = read_hashes(candidate)
|
||||
missing = expected.keys() - actual.keys()
|
||||
extra = actual.keys() - expected.keys()
|
||||
mismatches = {field: 0 for field in ("mesh", "payload", "topology", "density_processed")}
|
||||
for key in expected.keys() & actual.keys():
|
||||
fields = ["mesh", "payload", "topology"]
|
||||
if key in candidate_generated:
|
||||
fields.append("density")
|
||||
for field in fields:
|
||||
if expected[key][field] != actual[key][field]:
|
||||
mismatches["density_processed" if field == "density" else field] += 1
|
||||
return {
|
||||
"reference": str(reference.relative_to(ROOT)),
|
||||
"candidate": str(candidate.relative_to(ROOT)),
|
||||
"reference_tiles": len(expected), "candidate_tiles": len(actual),
|
||||
"missing_keys": len(missing), "extra_keys": len(extra),
|
||||
"mismatches": mismatches,
|
||||
"equal": not missing and not extra and not any(mismatches.values()),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not DATA.is_file() or not BIN.is_file():
|
||||
raise SystemExit("missing owner world or L74 Clang headless binary")
|
||||
references, candidates, comparisons = [], [], []
|
||||
for view in range(4):
|
||||
for position, offset in enumerate(OFFSETS):
|
||||
reference = run("reference", view, position, offset)
|
||||
candidate = run("skip", view, position, offset)
|
||||
comparison = compare(ROOT / reference["log"], ROOT / candidate["log"])
|
||||
comparison.update(strate=view, position=position)
|
||||
references.append(reference)
|
||||
candidates.append(candidate)
|
||||
comparisons.append(comparison)
|
||||
print("[L74HashSweepCompare] " + json.dumps(comparison, sort_keys=True), flush=True)
|
||||
if not comparison["equal"]:
|
||||
raise SystemExit(f"whole-tile proof falsified; see {comparison['candidate']}")
|
||||
summaries = []
|
||||
for view in range(4):
|
||||
rows = [row for row in candidates if row["strate"] == view]
|
||||
skips = sum(row["whole_tile_skips"] for row in rows)
|
||||
tiles = sum(row["tiles"] for row in rows)
|
||||
costs = [row["classification_worker_ms"] for row in rows]
|
||||
summaries.append({
|
||||
"strate": view, "camera_positions": len(rows), "hash_comparisons": len(rows),
|
||||
"violations": sum(not row["equal"] for row in comparisons
|
||||
if row["strate"] == view),
|
||||
"whole_tile_proof_hits": skips, "tiles_visited": tiles,
|
||||
"proof_hit_pct": round(100.0 * skips / max(tiles, 1), 2),
|
||||
"classifier_ms_p50": round(statistics.median(costs), 3),
|
||||
"classifier_ms_p95": round(sorted(costs)[int(0.95 * (len(costs) - 1))], 3),
|
||||
})
|
||||
result = {"runs": {"reference": references, "skip": candidates},
|
||||
"comparisons": comparisons, "summary": summaries}
|
||||
(OUT / "hash-sweep.json").write_text(json.dumps(result, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
for row in summaries:
|
||||
print("[L74HashSweepSummary] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run three interleaved clean full-view pairs on L73 and L74 headless builds."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l74/logs/timing"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BINARIES = {
|
||||
"before": ROOT / "Saved/Linux/l73/bin/VoxelForgeCoreHeadless-l73-clang",
|
||||
"after": ROOT / "Saved/Linux/l74/bin/VoxelForgeCoreHeadless-l74-clang",
|
||||
}
|
||||
CAMERAS = {
|
||||
0: (8.0, 16.0, -372.2626953125),
|
||||
1: (25.8369140625, 3.19921875, -737.0),
|
||||
2: (19.166015625, -14.2001953125, -753.3740234375),
|
||||
3: (-20.0625, -5.068359375, -900.353515625),
|
||||
}
|
||||
ORDERS = (("before", "after"), ("after", "before"), ("before", "after"))
|
||||
PIPELINE = re.compile(r"\[StreamPipeline\] (.*)")
|
||||
PAIR = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
|
||||
|
||||
def run(strate: int, pair: int, mode: str) -> dict:
|
||||
label = f"strate{strate}-pair{pair}-{mode}"
|
||||
log = OUT / f"{label}.log"
|
||||
camera = ",".join(f"{value:.12f}" for value in CAMERAS[strate]) + ",0,0"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout",
|
||||
"1740s", str(BINARIES[mode]), str(DATA), "--world-assembler-prototype",
|
||||
"--stream-benchmark", "--stream-pipeline", "--force-generate", "--workers=8",
|
||||
"--clip-radius=3", "--max-clip-level=8", "--max-level=8", f"--camera={camera}",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as output:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT,
|
||||
check=False, timeout=1740)
|
||||
content = log.read_text(encoding="utf-8", errors="replace")
|
||||
match = PIPELINE.search(content)
|
||||
if result.returncode != 0 or match is None:
|
||||
raise SystemExit(f"timing run failed or missing StreamPipeline summary; see {log}")
|
||||
pipeline = dict(PAIR.findall(match.group(1)))
|
||||
row = {
|
||||
"strate": strate, "pair": pair, "mode": mode, "exit_code": result.returncode,
|
||||
"process_wall_s": round(time.monotonic() - started, 3),
|
||||
"whole_view_ready_s": float(pipeline["whole_view_ready"].removesuffix("s")),
|
||||
"stream_wall_s": float(pipeline["stream_wall"].removesuffix("s")),
|
||||
"whole_tiles": int(pipeline["whole_tiles"].split("/")[0]),
|
||||
"source_stamp": re.search(r"\[VoxelForgeBuild\] source_stamp=(\S+)", content),
|
||||
"log": str(log.relative_to(ROOT)),
|
||||
}
|
||||
row["source_stamp"] = row["source_stamp"].group(1) if row["source_stamp"] else None
|
||||
print("[L74TimingPairRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def main() -> int:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
missing = [str(path) for path in [DATA, *BINARIES.values()] if not path.is_file()]
|
||||
if missing:
|
||||
raise SystemExit("missing input: " + ", ".join(missing))
|
||||
rows = []
|
||||
for strate in range(4):
|
||||
for pair, order in enumerate(ORDERS, 1):
|
||||
for mode in order:
|
||||
rows.append(run(strate, pair, mode))
|
||||
summary = []
|
||||
for strate in range(4):
|
||||
group = [row for row in rows if row["strate"] == strate]
|
||||
before = [row["whole_view_ready_s"] for row in group if row["mode"] == "before"]
|
||||
after = [row["whole_view_ready_s"] for row in group if row["mode"] == "after"]
|
||||
summary.append({"strate": strate, "pairs": len(before),
|
||||
"before_s": [round(value, 3) for value in before],
|
||||
"after_s": [round(value, 3) for value in after],
|
||||
"before_median_s": round(statistics.median(before), 3),
|
||||
"after_median_s": round(statistics.median(after), 3),
|
||||
"change_pct": round(100.0 * (statistics.median(after)
|
||||
/ statistics.median(before) - 1.0), 2)})
|
||||
result = {"runs": rows, "summary": summary}
|
||||
(OUT / "summary.json").write_text(json.dumps(result, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
for row in summary:
|
||||
print("[L74TimingPairSummary] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user