diff --git a/Standalone/Research/WorldAssemblerM3/Linux107/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux107/REPORT.md new file mode 100644 index 0000000..7f36519 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux107/REPORT.md @@ -0,0 +1,47 @@ +# Linux 107: base-shape startup + +## Result + +Kept the exact startup overlap and parallel route check. Across five interleaved W8 pairs, whole-view readiness improved by a paired median 35–55 ms (1.6–2.2%) in all four strates. W12 was mostly flat outside SW. No passage support rules or density predicates changed. + +## Critical path + +In FI W8, base compile ran from 59.407 to 224.529 ms. LOD0–1 admission followed at 224.580 ms; the first tile was ready at 246 ms, LOD0–1 drained at 1.302 s, and LOD2 completed at 1.397 s. Base shape remains a serial prefix before LOD0–1 work. + +Route validation does not supply tile data; it is a hard compile/publication gate. Its exact density samples now run in parallel, then failures are reduced in canonical route, segment, and sample order. It still gates compile completion. The surface-distance band is already parallel and is read by exact density queries for candidate nodes, so this round leaves it eager. Owner-shape extraction and sign-edge repair were already parallel. Packed intervals now overlap X/Y candidate sweeps and passage prep with split worker budgets and disjoint outputs. + +## Substage snapshots + +FI/TN W8/W12 profile traces, L106 → L107, milliseconds: + +| Stage | L106 | L107 | +|---|---:|---:| +| Owner shapes | 21.8–23.2 | 22.3–25.1 | +| Packed + passage prep + candidates on path | 56.5–59.5 sequential | 45.3–54.5 overlapped | +| Detail band | 49.7–83.3 | 54.2–77.5 | +| Sign-edge repair | 9.4–10.3 | 9.7–11.6 | +| Route validation | 25.7–26.9 | 18.4–18.6 | +| Base compile | 173–205 | 157–189 | + +## Readiness + +Each value is the median of five complete runs. Parentheses give the median of the five matched after-minus-before pairs, in milliseconds. Negative is faster. + +| Stratum | W | Near LOD0–2 median (s) | Whole-view median (s) | +|---|---:|---:|---:| +| SW | 8 | 0.757 → 0.752 (−9) | 1.927 → 1.867 (−42) | +| SW | 12 | 0.769 → 0.736 (−43) | 1.810 → 1.771 (−39) | +| FI | 8 | 1.553 → 1.537 (−10) | 2.551 → 2.565 (−55) | +| FI | 12 | 1.436 → 1.443 (+6) | 2.393 → 2.389 (−6) | +| TN | 8 | 1.461 → 1.492 (+31) | 2.396 → 2.425 (−39) | +| TN | 12 | 1.314 → 1.327 (+18) | 2.174 → 2.192 (+19) | +| VS | 8 | 1.046 → 1.049 (−13) | 2.018 → 1.971 (−35) | +| VS | 12 | 0.877 → 0.952 (+11) | 1.735 → 1.827 (−2) | + +## Checks + +- Exact hashes: 24/24 cases match L106 per-tile and whole-view hashes (SW/FI/TN/VS × Clang/GCC × W1/W8/W12); zero mismatches. +- Fast tier: `SUMMARY PASS` (269 pass, five existing diagnostics). +- Linux UE build: succeeded in 68.9 s with `-NoUBA -MaxParallelActions=4`. +- UE arrivals: 3 FI and 3 SW succeeded, with no world-test failures. Unreal ran with taskset and timeout, without `prlimit`. +- Logs and machine-readable results: `Saved/Linux/l107/`. Candidate source stamp: `5386228461b81b97f5649f6dc4beab5e3b9b8fe38ea706064a825d3ac77e65d3`. diff --git a/Standalone/Research/WorldAssemblerM3/Linux107/RunL107Hashes.py b/Standalone/Research/WorldAssemblerM3/Linux107/RunL107Hashes.py new file mode 100644 index 0000000..fdb4c8d --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux107/RunL107Hashes.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Compare L107 exact tile hashes with L106 across all strata, compilers and workers.""" +from __future__ import annotations + +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l107/hashes" +DATA = ROOT / "Standalone/data/owner_world.json" +OLD = ROOT / "Saved/Linux/l106/hashes" +BINARIES = { + "clang": ROOT / "Saved/Linux/l107/bin/VoxelForgeCoreHeadless-l107-clang", + "gcc": ROOT / "Saved/Linux/l107/bin/VoxelForgeCoreHeadless-l107-gcc", +} +CAMERAS = { + "sw": ((8.0, 16.0, -372.2626953125), 2850), + "fi": ((25.8369140625, 3.19921875, -737.0), 2249), + "tn": ((19.166015625, -14.2001953125, -753.3740234375), 2233), + "vs": ((-20.0625, -5.068359375, -900.353515625), 2329), +} +TILE = re.compile( + r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) " + r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) " + r"topology=(0x[0-9a-f]+)$", re.M) +WHOLE = re.compile( + r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) " + r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M) + + +def parse(path: Path) -> tuple[dict, tuple[str, ...], str]: + content = path.read_text(encoding="utf-8", errors="replace") + tiles = {(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))): + m.group(5, 6, 7, 8) for m in TILE.finditer(content)} + whole, stamp = WHOLE.search(content), STAMP.search(content) + if whole is None or stamp is None or len(tiles) != int(whole.group(1)): + raise RuntimeError(f"incomplete hash output: {path}") + return tiles, whole.groups(), stamp.group(1) + + +def run(stratum: str, compiler: str, workers: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + log = OUT / f"{stratum}-{compiler}-w{workers}.log" + camera, expected_tiles = CAMERAS[stratum] + camera_arg = ",".join(f"{v:.12f}" for v in camera) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARIES[compiler]), str(DATA), "--world-assembler-prototype", + "--stream-benchmark", "--stream-pipeline", f"--workers={workers}", + "--clip-radius=3", "--max-clip-level=8", "--max-level=8", + f"--camera={camera_arg}", "--stream-hash-tiles", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + process = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + if process.returncode: + raise RuntimeError(f"hash process exited {process.returncode}: {log}") + tiles, whole, stamp = parse(log) + if len(tiles) != expected_tiles: + raise RuntimeError(f"expected {expected_tiles} tiles, got {len(tiles)}: {log}") + old_log = OLD / f"{stratum}-{compiler}-w{workers}.log" + old_tiles, old_whole, _old_stamp = parse(old_log) + clang_reference, clang_whole, _ = parse(OLD / f"{stratum}-clang-w1.log") + + def difference(reference: dict, candidate: dict) -> tuple[int, int]: + keys = set(reference) | set(candidate) + mismatches = sum(reference.get(key) != candidate.get(key) for key in keys) + return mismatches, len(keys) + + old_mismatches, old_keys = difference(old_tiles, tiles) + cross_mismatches, cross_keys = difference(clang_reference, tiles) + row = { + "stratum": stratum, "compiler": compiler, "workers": workers, + "source_stamp": stamp, "tiles": len(tiles), + "l106_same_cell_tile_mismatches": old_mismatches, + "l106_same_cell_key_count": old_keys, + "l106_same_cell_whole_hash_match": whole == old_whole, + "clang_w1_tile_mismatches": cross_mismatches, + "clang_w1_key_count": cross_keys, + "clang_w1_whole_hash_match": whole == clang_whole, + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT)), + } + print("[L107HashRun] " + json.dumps(row, sort_keys=True), flush=True) + row["tile_hashes"] = {"|".join(map(str, key)): value for key, value in tiles.items()} + return row + + +def main() -> int: + for path in (*BINARIES.values(), DATA): + if not path.is_file(): + raise SystemExit(f"missing input: {path}") + rows = [] + for stratum in CAMERAS: + for compiler in BINARIES: + for workers in (1, 8, 12): + rows.append(run(stratum, compiler, workers)) + (OUT / "runs-partial.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + failures = [row for row in rows if row["l106_same_cell_tile_mismatches"] + or not row["l106_same_cell_whole_hash_match"] + or row["clang_w1_tile_mismatches"] + or not row["clang_w1_whole_hash_match"]] + summary = {"runs": [{key: value for key, value in row.items() if key != "tile_hashes"} + for row in rows], "failed_runs": len(failures)} + (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", + encoding="utf-8") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux107/RunL107Readiness.py b/Standalone/Research/WorldAssemblerM3/Linux107/RunL107Readiness.py new file mode 100644 index 0000000..7f0080a --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux107/RunL107Readiness.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""L80-protocol L106/L107 near and whole-view readiness pairs.""" +from __future__ import annotations + +import json +import re +import statistics +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l107/readiness" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARIES = { + "before": ROOT / "Saved/Linux/l106/bin/VoxelForgeCoreHeadless-l106-clang", + "after": ROOT / "Saved/Linux/l107/bin/VoxelForgeCoreHeadless-l107-clang", +} +STRATA = { + "sw": {"camera": (8.0, 16.0, -372.2626953125), "near": 1408, "whole": 2850}, + "fi": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249}, + "tn": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233}, + "vs": {"camera": (-20.0625, -5.068359375, -900.353515625), "near": 1344, "whole": 2329}, +} +NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") + + +def run(stratum: str, workers: int, build: str, phase: str, replicate: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + suffix = f"-r{replicate}" if phase == "measured" else "" + log = OUT / f"{stratum}-w{workers}-{build}-{phase}{suffix}.log" + camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARIES[build]), str(DATA), "--world-assembler-prototype", + "--stream-benchmark", "--stream-pipeline", f"--workers={workers}", + "--clip-radius=3", "--max-clip-level=8", "--max-level=8", + f"--camera={camera}", "--profile-lod-work", "--stream-scheduler-policy=3", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + process = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + content = log.read_text(encoding="utf-8", errors="replace") + near, pipeline, config, stamp = (NEAR.search(content), PIPELINE.search(content), + CONFIG.search(content), STAMP.search(content)) + if process.returncode or not all((near, pipeline, config, stamp)): + raise RuntimeError(f"readiness run failed or missing marker: {log}; exit={process.returncode}") + expected = STRATA[stratum] + config_fields = dict(FIELDS.findall(config.group(1))) + pipeline_fields = dict(FIELDS.findall(pipeline.group(1))) + if int(config_fields["workers"]) != workers: + raise RuntimeError(f"worker mismatch: {log}") + if (int(near.group(1)), int(near.group(2))) != (expected["near"], expected["near"]): + raise RuntimeError(f"incomplete LOD0-2 set: {log}: {near.group(1)}/{near.group(2)}") + whole_done, whole_total = map(int, pipeline_fields["whole_tiles"].split("/")) + if (whole_done, whole_total) != (expected["whole"], expected["whole"]): + raise RuntimeError(f"incomplete whole view: {log}: {whole_done}/{whole_total}") + row = { + "stratum": stratum, "workers": workers, "build": build, + "phase": phase, "replicate": replicate, "source_stamp": stamp.group(1), + "near_tiles": expected["near"], "whole_tiles": whole_done, + "lod02_ready_s": float(near.group(3)), + "whole_view_ready_s": float(pipeline_fields["whole_view_ready"].removesuffix("s")), + "shape_s": float(pipeline_fields["shape"].removesuffix("s")), + "far_shape_s": float(pipeline_fields["far_shape"].removesuffix("s")), + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT)), + } + print("[L107ReadinessRun] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + for path in (*BINARIES.values(), DATA): + if not path.is_file(): + raise SystemExit(f"missing input: {path}") + cells = [(stratum, workers) for stratum in ("vs", "tn", "fi", "sw") + for workers in (12, 8)] + for stratum, workers in cells: + for build in ("before", "after"): + run(stratum, workers, build, "warmup", 0) + orders = (("before", "after"), ("after", "before"), ("before", "after"), + ("after", "before"), ("before", "after")) + rows = [] + for stratum, workers in cells: + for replicate, order in enumerate(orders, 1): + for build in order: + rows.append(run(stratum, workers, build, "measured", replicate)) + (OUT / "runs-partial.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + summary = [] + for stratum in STRATA: + for workers in (8, 12): + group = [row for row in rows if row["stratum"] == stratum + and row["workers"] == workers] + item = {"stratum": stratum, "workers": workers, "measured_runs_per_build": 5} + for build in ("before", "after"): + subset = [row for row in group if row["build"] == build] + for metric in ("lod02_ready_s", "whole_view_ready_s"): + values = [row[metric] for row in subset] + item[f"{build}_{metric}"] = values + item[f"{build}_{metric}_median"] = statistics.median(values) + for metric in ("lod02_ready_s", "whole_view_ready_s"): + item[f"{metric}_delta_pct"] = 100.0 * ( + item[f"after_{metric}_median"] / item[f"before_{metric}_median"] - 1.0) + summary.append(item) + (OUT / "summary.json").write_text( + json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n", + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux107/RunL107TimerTraces.py b/Standalone/Research/WorldAssemblerM3/Linux107/RunL107TimerTraces.py new file mode 100644 index 0000000..26c3313 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux107/RunL107TimerTraces.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Capture comparable FI/TN L106 and L107 shape-build timelines.""" +from __future__ import annotations + +import json +import argparse +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l107/timeline" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARIES = { + "before": ROOT / "Saved/Linux/l106/bin/VoxelForgeCoreHeadless-l106-clang", + "after": ROOT / "Saved/Linux/l107/bin/VoxelForgeCoreHeadless-l107-clang", +} +STRATA = { + "fi": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249}, + "tn": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233}, +} +NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") + + +def run(stratum: str, workers: int, build: str) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + log = OUT / f"{build}-{stratum}-w{workers}.log" + camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARIES[build]), str(DATA), "--world-assembler-prototype", "--stream-benchmark", + "--stream-pipeline", f"--workers={workers}", "--clip-radius=3", "--max-clip-level=8", + "--max-level=8", f"--camera={camera}", "--profile-lod-work", "--stream-scheduler-policy=3", + "--cold-worker-timeline", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + content = log.read_text(encoding="utf-8", errors="replace") + near, pipeline = NEAR.search(content), PIPELINE.search(content) + config, stamp = CONFIG.search(content), STAMP.search(content) + shape = re.search(r"^\[ShapeBreakdown\] (.*)$", content, re.M) + profile = re.search(r"^\[ShapeProfile\] (.*)$", content, re.M) + substage = re.search(r"^\[ShapeSubstage\] (.*)$", content, re.M) + overlap = re.search(r"^\[ShapeStartupOverlap\] (.*)$", content, re.M) + if result.returncode or not all((near, pipeline, config, stamp, shape, profile, substage)): + raise RuntimeError(f"timer trace failed or missing markers: {log}; exit={result.returncode}") + config_fields = dict(FIELDS.findall(config.group(1))) + pipeline_fields = dict(FIELDS.findall(pipeline.group(1))) + expected = STRATA[stratum] + if int(config_fields["workers"]) != workers: + raise RuntimeError(f"worker mismatch: {log}") + if (int(near.group(1)), int(near.group(2))) != (expected["near"], expected["near"]): + raise RuntimeError(f"incomplete near tile set: {log}") + if tuple(map(int, pipeline_fields["whole_tiles"].split("/"))) != (expected["whole"], expected["whole"]): + raise RuntimeError(f"incomplete whole tile set: {log}") + row = { + "build": build, "stratum": stratum, "workers": workers, + "source_stamp": stamp.group(1), + "near_ready_s": float(near.group(3)), + "whole_view_ready_s": float(pipeline_fields["whole_view_ready"].removesuffix("s")), + "base_shape_s": float(pipeline_fields["shape"].removesuffix("s")), + "shape_breakdown": dict(FIELDS.findall(shape.group(1))), + "shape_profile": dict(FIELDS.findall(profile.group(1))), + "shape_substage": dict(FIELDS.findall(substage.group(1))), + "shape_startup_overlap": dict(FIELDS.findall(overlap.group(1))) if overlap else None, + "log": str(log.relative_to(ROOT)), + "runner_wall_s": round(time.monotonic() - started, 3), + } + print("[L107TimerTrace] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--build", choices=("before", "after", "both"), default="both") + args = parser.parse_args() + builds = ("before", "after") if args.build == "both" else (args.build,) + for build in builds: + binary = BINARIES[build] + if not binary.is_file(): + raise SystemExit(f"missing input binary: {binary}") + rows = [run(stratum, workers, build) + for stratum in ("fi", "tn") for workers in (8, 12) for build in builds] + (OUT / f"runs-{args.build}.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux107/RunL107UnrealArrival.py b/Standalone/Research/WorldAssemblerM3/Linux107/RunL107UnrealArrival.py new file mode 100644 index 0000000..fa9ba46 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux107/RunL107UnrealArrival.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 +"""Run one pinned null-RHI SW/FI arrival for Linux 107.""" +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +HOST = Path("/home/jahni/VFHost") +HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l107" +HOST_SAVED = HOST / "Saved/Linux/l107" +EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd") +OUT = ROOT / "Saved/Linux/l107/unreal-arrival" +SPAWNS_M = { + "FI": (25.8369140625, 3.19921875, -737.0), + "SW": (25.8369140625, 3.19921875, -200.0), +} +MOVE_START_S = 20.0 +MOVE_DURATION_S = 12.0 +EXIT_S = MOVE_START_S + MOVE_DURATION_S + 5.0 +RUN_TIMEOUT_S = 1740 + + +def read_events(path: Path) -> list[dict]: + events = [] + if path.is_file(): + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith("{"): + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + pass + return events + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + return ordered[min(len(ordered) - 1, int(round((len(ordered) - 1) * fraction)))] + + +def sample_unreal_peak_rss() -> tuple[int, int] | None: + executable = str(EDITOR).encode() + project = str(HOST / "VFHost.uproject").encode() + peak: tuple[int, int] | None = None + for process in Path("/proc").glob("[0-9]*"): + try: + command_line = (process / "cmdline").read_bytes() + if executable not in command_line or project not in command_line: + continue + status = (process / "status").read_text(encoding="ascii", errors="ignore") + match = re.search(r"^VmHWM:\s+(\d+)\s+kB$", status, re.MULTILINE) + if match: + rss_kib = int(match.group(1)) + if peak is None or rss_kib > peak[0]: + peak = (rss_kib, int(process.name)) + except (FileNotFoundError, PermissionError, ProcessLookupError, OSError): + continue + return peak + + +def endphysics_gap(stdout: Path) -> tuple[float | None, int | None]: + release_times: dict[int, dict[int, datetime]] = {} + pattern = re.compile( + r"^\[(\d{4}\.\d{2}\.\d{2}-\d{2}\.\d{2}\.\d{2}:\d{3})\].*" + r"LogTick: tick\s+(\d+)\s+-+\s+Release tick group ([34])") + with stdout.open(encoding="utf-8", errors="replace") as stream: + for line in stream: + match = pattern.match(line) + if not match: + continue + timestamp = datetime.strptime(match.group(1), "%Y.%m.%d-%H.%M.%S:%f") + frame = int(match.group(2)) + group = int(match.group(3)) + release_times.setdefault(frame, {})[group] = timestamp + gaps = [(round((times[4] - times[3]).total_seconds() * 1000.0, 3), frame) + for frame, times in release_times.items() if 3 in times and 4 in times] + return max(gaps, default=(None, None)) + + +def run(stratum: str, repeat: int, phase: str, ready_depth: int, + background_cap: int, worker_reserve: int, helper_background: int, + tile_background_low: int, tile_background_normal: int, startup_boost: int, + tick_logs: bool, insights: bool) -> dict: + label = (f"{phase}-{stratum.lower()}-rd{ready_depth}-bg{background_cap}" + f"-reserve{worker_reserve}-helperbg{helper_background}" + f"-tilelow{tile_background_low}-tilenormal{tile_background_normal}" + f"-ticklog{int(tick_logs)}" + f"-boost{startup_boost}-{repeat}") + trace = OUT / f"{label}.trace.jsonl" + utrace = OUT / f"{label}.utrace" + abslog = OUT / f"{label}.log" + stdout = OUT / f"{label}.stdout.log" + output = HOST_PLUGIN_SAVED / "output" / label + userdir = HOST_SAVED / "user" / label + zendata = HOST_SAVED / "zen" / label + ddc = HOST_SAVED / "DDC" + for directory in (OUT, output, userdir, zendata, ddc): + directory.mkdir(parents=True, exist_ok=True) + trace.unlink(missing_ok=True) + utrace.unlink(missing_ok=True) + spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWNS_M[stratum]) + command = [ + str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", + "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", + "-notraceserver", "-nocrashreports", + "-DisablePlugins=AndroidFileServer", + "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", + f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", + "-archetype=TunnelNetwork", f"-slot={1 if stratum == 'FI' else 0}", + "-modes=export", "-gameconfig=1", "-coreworldtest=1", + "-ownerfourstrateplan=1", "-corevoxelsizecm=25", "-coreislandtest=0", + "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0", + "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0", + f"-out={output}", "-voxel.StreamingCoarseFirst=1", + "-voxel.StreamingDynamicNearReserve=1", + "-voxel.StreamingLegacyPendingResultCap=0", + "-voxel.CoreRegionConcurrentAdmission=1", + "-voxel.StreamingResultBacklogCap=256", + f"-voxel.StreamingReadyQueueDepth={ready_depth}", + f"-voxel.StreamingBackgroundWorkerCap={background_cap}", + f"-voxel.StreamingBackgroundWorkerReserve={worker_reserve}", + f"-voxel.CoreParallelForBackgroundPriority={helper_background}", + f"-voxel.StreamingTileBackgroundLow={tile_background_low}", + f"-voxel.StreamingTileBackgroundNormal={tile_background_normal}", + f"-voxel.TestLogTickFunctions={int(tick_logs)}", + "-voxel.CoreColumnCacheDirectMapped=0", "-voxel.CoreMCGradientMemoization=1", + "-voxel.StreamingCollisionLookahead=1", + f"-voxel.StreamingStartupApplyBoost={startup_boost}", + "-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=140.0", + f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}", + f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}", + f"-voxel.TestExitSeconds={EXIT_S:.1f}", "-voxel.TestMoveDirection=1,0,0", + "-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0", + "-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}", + "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", + "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", + f"-abslog={abslog}", + ] + if insights: + command += ["-trace=cpu,task,frame", "-statnamedevents", + f"-tracefile={utrace}", "-tracefiletrunc"] + wrapped = ["taskset", "-c", "0-11", "timeout", f"{RUN_TIMEOUT_S}s", *command] + started = time.monotonic() + with stdout.open("w", encoding="utf-8") as stream: + process = subprocess.Popen(wrapped, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT) + peak_rss = None + while process.poll() is None: + observed = sample_unreal_peak_rss() + if observed and (peak_rss is None or observed[0] > peak_rss[0]): + peak_rss = observed + time.sleep(0.25) + exit_code = process.wait() + observed = sample_unreal_peak_rss() + if observed and (peak_rss is None or observed[0] > peak_rss[0]): + peak_rss = observed + + events = read_events(trace) + worker_backend_config = next( + (e for e in events if e.get("type") == "worker_backend_config"), {}) + worker_cap_readback = next( + (e for e in events if e.get("type") == "streaming_worker_cap_readback"), {}) + samples = [e for e in events if e.get("type") == "streaming_sample"] + arrival = [e for e in samples if float(e.get("t_s", 0.0)) < MOVE_START_S] + full = next((e for e in arrival + if sum(e.get("desired_by_lod", [0] * 9)[:3]) > 0 + and sum(e.get("settled_by_lod", [0] * 9)[:3]) + >= sum(e.get("desired_by_lod", [0] * 9)[:3])), None) + first_admission = next((e for e in events if e.get("type") == "scheduler_admission" + and e.get("desired_rebuilt") == 1), None) + request_events = [e for e in events if e.get("type") == "request"] + initial_request_events = [e for e in request_events if e.get("epoch") == 0] + tile_by_ordinal = {e.get("request_ordinal"): e for e in events + if e.get("type") == "tile"} + collision_count = sum(1 for e in events if e.get("type") == "collision_ready" + and e.get("level", 9) <= 2 + and full and e.get("t_s", 1e30) <= full["t_s"]) + initial_tiles = [tile_by_ordinal[r.get("request_ordinal")] for r in initial_request_events + if full and r.get("t_s", 1e30) <= full["t_s"] + and r.get("request_ordinal") in tile_by_ordinal + and tile_by_ordinal[r["request_ordinal"]].get("level", 9) <= 2] + burst_start_s = float(first_admission["t_s"]) if first_admission else -1.0 + burst_end_s = float(full["t_s"]) if full else -1.0 + work_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "game_thread_work_frame"} + work_values = [e for e in work_by_frame.values() + if burst_start_s <= float(e.get("t_s", -1.0)) <= burst_end_s] + arrival_frame_ids = {int(e.get("frame", -1)) for e in work_values} + scheduler_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "scheduler_admission"} + scheduler_values = [e for e in events if e.get("type") == "scheduler_admission" + and burst_start_s <= float(e.get("t_s", -1.0)) <= burst_end_s] + apply_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "apply_frame"} + commandlet_tick_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "commandlet_world_tick_frame"} + garbage_collect_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "garbage_collect"} + # apply_frame N+1 records the interval containing AVoxelWorld::Tick N. Its apply_ms + # belongs to Tick N+1, so use apply_by_frame[N] for the same-tick apply breakdown. + near_intervals = [apply_by_frame[frame + 1] for frame in arrival_frame_ids + if frame + 1 in apply_by_frame] + rmc_by_frame: dict[int, dict[str, float]] = {} + for event in events: + frame = int(event.get("frame", -1)) + if event.get("type") != "rmc_submit" or frame not in arrival_frame_ids: + continue + values = rmc_by_frame.setdefault(frame, { + "component_acquire_ms": 0.0, + "create_section_group_ms": 0.0, + "create_sections_ms": 0.0, + "collision_handoff_ms": 0.0, + }) + for key in values: + values[key] += float(event.get(key, 0.0)) + collision_completion_by_frame: dict[int, float] = {} + for event in events: + frame = int(event.get("frame", -1)) + if event.get("type") == "collision_completion" and frame in arrival_frame_ids: + collision_completion_by_frame[frame] = collision_completion_by_frame.get(frame, 0.0) \ + + float(event.get("handler_ms", 0.0)) + near_requests = [r for r in request_events if full and r.get("t_s", 1e30) <= full["t_s"] + and r.get("level", 9) <= 2] + initial_lod0_counts = Counter( + tuple(r.get("tile", ())) for r in near_requests if r.get("level") == 0) + repeated_lod0_keys = [ + {"tile": list(tile), "requests": count} + for tile, count in sorted(initial_lod0_counts.items()) if count > 1] + initial_scheduler_frames = [e for e in events + if e.get("type") == "scheduler_admission" and full + and e.get("t_s", 1e30) <= full["t_s"]] + worst_interval = max(near_intervals, key=lambda e: float(e.get("frame_ms", 0.0)), default=None) + worst_interval_frame_id = int(worst_interval.get("frame", -1)) if worst_interval else -1 + worst_work_frame_id = worst_interval_frame_id - 1 + worst_work = work_by_frame.get(worst_work_frame_id, {}) + worst_apply = apply_by_frame.get(worst_work_frame_id, {}) + worst_scheduler = scheduler_by_frame.get(worst_work_frame_id, {}) + worst_rmc = rmc_by_frame.get(worst_work_frame_id, {}) + worst_collision_completion_ms = collision_completion_by_frame.get(worst_work_frame_id, 0.0) + rmc_values = [rmc_by_frame.get(frame, {}) for frame in arrival_frame_ids] + apply_values = [apply_by_frame[frame] for frame in arrival_frame_ids + if frame in apply_by_frame] + commandlet_tick_values = [commandlet_tick_by_frame[frame] for frame in arrival_frame_ids + if frame in commandlet_tick_by_frame] + garbage_collect_values = [garbage_collect_by_frame[frame] for frame in arrival_frame_ids + if frame in garbage_collect_by_frame] + + def work_stat(name: str, fraction: float | None = None) -> float | None: + values = [float(e.get(name, 0.0)) for e in work_values] + return percentile(values, fraction) if fraction is not None else max(values, default=0.0) + + def scheduler_stat(name: str, fraction: float | None = None) -> float | None: + values = [float(e.get(name, 0.0)) for e in scheduler_values] + return percentile(values, fraction) if fraction is not None else max(values, default=0.0) + + def rmc_stat(name: str, fraction: float | None = None) -> float | None: + values = [float(e.get(name, 0.0)) for e in rmc_values] + return percentile(values, fraction) if fraction is not None else max(values, default=0.0) + + def commandlet_stat(name: str, fraction: float | None = None) -> float | None: + values = [float(e.get(name, 0.0)) for e in commandlet_tick_values] + return percentile(values, fraction) if fraction is not None else max(values, default=0.0) + + row = { + "label": label, "stratum": stratum, "repeat": repeat, "phase": phase, + "ready_queue_depth": ready_depth, + "requested_background_worker_cap": background_cap, + "background_worker_cap": worker_cap_readback.get( + "streaming_background_worker_cap", background_cap), + "requested_background_worker_reserve": worker_reserve, + "background_worker_reserve": worker_backend_config.get( + "streaming_background_worker_reserve", worker_cap_readback.get( + "background_worker_reserve", worker_reserve)), + "requested_core_parallel_for_background_priority": helper_background, + "core_parallel_for_background_priority": worker_backend_config.get( + "core_parallel_for_background_priority", helper_background), + "requested_streaming_tile_background_low": tile_background_low, + "streaming_tile_background_low": worker_backend_config.get( + "streaming_tile_background_low", tile_background_low), + "requested_streaming_tile_background_normal": tile_background_normal, + "streaming_tile_background_normal": worker_backend_config.get( + "streaming_tile_background_normal", tile_background_normal), + "effective_tile_slots": worker_backend_config.get("effective_tile_slots"), + "background_worker_threads": worker_backend_config.get("taskgraph_background_threads"), + "startup_apply_boost": startup_boost, "insights": insights, + "engine_tick_logs": tick_logs, + "exit_code": exit_code, + "launch_wrapper": "taskset -c 0-11 timeout 1740s (no address-space limit)", + "peak_rss_kib": peak_rss[0] if peak_rss else None, + "peak_rss_mib": round(peak_rss[0] / 1024.0, 1) if peak_rss else None, + "peak_rss_pid": peak_rss[1] if peak_rss else None, + "endphysics_to_postphysics_gap_ms_max": None, + "endphysics_gap_frame": None, + "wall_s": round(time.monotonic() - started, 3), + "desired_t_s": next((e.get("t_s") for e in events if e.get("type") == "desired_set"), None), + "first_admission_t_s": first_admission.get("t_s") if first_admission else None, + "lod02_full_settled_t_s": full.get("t_s") if full else None, + "near_settle_from_first_admission_s": round( + float(full["t_s"]) - float(first_admission["t_s"]), 4) + if full and first_admission else None, + "desired_by_lod": full.get("desired_by_lod", [])[:3] if full else None, + "settled_by_lod": full.get("settled_by_lod", [])[:3] if full else None, + "near_admitted_tiles": len(near_requests), + "initial_lod0_admissions": sum(initial_lod0_counts.values()), + "repeated_lod0_request_keys_before_settle": repeated_lod0_keys, + "peak_result_backlog_before_settle": max( + (int(e.get("result_backlog", 0)) for e in initial_scheduler_frames), default=0), + "last_admission_t_s": max((e.get("t_s", 0.0) for e in near_requests), default=None), + "near_tile_events": len(initial_tiles), + "near_worker_seconds": round(sum(e.get("worker_task_s", 0.0) for e in initial_tiles), 4), + "near_core_seconds": round(sum(e.get("core_generate_s", 0.0) for e in initial_tiles), 4), + "collision_ready_by_settle": collision_count, + "burst_frame_count": len(near_intervals), + "burst_frame_ms_p95": round(sorted(e.get("frame_ms", 0.0) for e in near_intervals)[ + min(len(near_intervals) - 1, int(.95 * (len(near_intervals) - 1)))], 3) + if near_intervals else None, + "burst_frame_ms_max": round(max((e.get("frame_ms", 0.0) for e in near_intervals), default=0.0), 3), + "burst_full_world_tick_ms_p99": round(commandlet_stat("world_tick_ms", .99) or 0.0, 3), + "burst_full_world_tick_ms_max": round(commandlet_stat("world_tick_ms") or 0.0, 3), + "burst_world_pre_actor_ms_p99": round(commandlet_stat("world_pre_actor_ms", .99) or 0.0, 3), + "burst_world_pre_actor_ms_max": round(commandlet_stat("world_pre_actor_ms") or 0.0, 3), + "burst_world_actor_phase_ms_p99": round(commandlet_stat("world_actor_phase_ms", .99) or 0.0, 3), + "burst_world_actor_phase_ms_max": round(commandlet_stat("world_actor_phase_ms") or 0.0, 3), + "burst_world_post_actor_ms_p99": round(commandlet_stat("world_post_actor_ms", .99) or 0.0, 3), + "burst_world_post_actor_ms_max": round(commandlet_stat("world_post_actor_ms") or 0.0, 3), + "burst_commandlet_readiness_check_ms_max": round( + commandlet_stat("readiness_check_ms") or 0.0, 3), + "burst_commandlet_sleep_ms_p99": round(commandlet_stat("sleep_ms", .99) or 0.0, 3), + "burst_commandlet_sleep_ms_max": round(commandlet_stat("sleep_ms") or 0.0, 3), + "burst_commandlet_loop_ms_max": round(commandlet_stat("loop_ms") or 0.0, 3), + "burst_gc_count": len(garbage_collect_values), + "burst_gc_duration_ms_max": round(max( + (float(e.get("duration_ms", 0.0)) for e in garbage_collect_values), default=0.0), 3), + "burst_max_meshes_per_frame": max((e.get("meshes_applied", 0) for e in apply_values), default=0), + "burst_max_apply_ms": round(max((e.get("apply_ms", 0.0) for e in apply_values), default=0.0), 3), + "burst_max_admit_per_frame": max((sum(e.get("submitted_by_lod", [0] * 9)[:3]) + for e in scheduler_values), default=0), + "burst_frame_timing_count": len(work_values), + "burst_active_generation_workers_p95": round(work_stat("active_generation_workers", .95) or 0.0, 3), + "burst_active_generation_workers_max": round(work_stat("active_generation_workers") or 0.0, 3), + "burst_active_tile_tasks_max": round(work_stat("active_tile_tasks") or 0.0, 3), + "burst_world_tick_ms_p99": round(work_stat("tick_ms", .99) or 0.0, 3), + "burst_world_tick_ms_max": round(work_stat("tick_ms") or 0.0, 3), + "burst_update_chunks_ms_p99": round(work_stat("update_chunks_ms", .99) or 0.0, 3), + "burst_update_chunks_ms_max": round(work_stat("update_chunks_ms") or 0.0, 3), + "burst_pending_drain_ms_p99": round(work_stat("pending_drain_ms", .99) or 0.0, 3), + "burst_pending_drain_ms_max": round(work_stat("pending_drain_ms") or 0.0, 3), + "burst_post_apply_callbacks_ms_p99": round(work_stat("post_apply_callbacks_ms", .99) or 0.0, 3), + "burst_post_apply_callbacks_ms_max": round(work_stat("post_apply_callbacks_ms") or 0.0, 3), + "burst_visibility_reconcile_ms_p99": round(work_stat("visibility_reconcile_ms", .99) or 0.0, 3), + "burst_visibility_reconcile_ms_max": round(work_stat("visibility_reconcile_ms") or 0.0, 3), + "burst_collision_probe_ms_max": round(work_stat("collision_probe_ms") or 0.0, 3), + "burst_visible_apply_probe_ms_max": round(work_stat("visible_apply_probe_ms") or 0.0, 3), + "burst_lod0_ready_ms_max": round(work_stat("lod0_ready_ms") or 0.0, 3), + "burst_steady_trace_ms_max": round(work_stat("steady_trace_ms") or 0.0, 3), + "burst_modification_ms_max": round(work_stat("modification_ms") or 0.0, 3), + "burst_lifecycle_audit_ms_p99": round(work_stat("lifecycle_audit_ms", .99) or 0.0, 3), + "burst_lifecycle_audit_ms_max": round(work_stat("lifecycle_audit_ms") or 0.0, 3), + "burst_collision_gate_ms_p99": round(work_stat("collision_gate_ms", .99) or 0.0, 3), + "burst_collision_gate_ms_max": round(work_stat("collision_gate_ms") or 0.0, 3), + "burst_unload_ms_p99": round(work_stat("unload_ms", .99) or 0.0, 3), + "burst_unload_ms_max": round(work_stat("unload_ms") or 0.0, 3), + "burst_trace_json_ms_p99": round(work_stat("trace_json_ms", .99) or 0.0, 3), + "burst_trace_json_ms_max": round(work_stat("trace_json_ms") or 0.0, 3), + "burst_desired_rebuild_ms_max": round(scheduler_stat("desired_rebuild_ms") or 0.0, 3), + "burst_rmc_create_group_ms_p99": round(rmc_stat("create_section_group_ms", .99) or 0.0, 3), + "burst_rmc_create_group_ms_max": round(rmc_stat("create_section_group_ms") or 0.0, 3), + "burst_rmc_collision_handoff_ms_max": round(rmc_stat("collision_handoff_ms") or 0.0, 3), + "burst_collision_completion_ms_max": round(max( + collision_completion_by_frame.values(), default=0.0), 3), + "worst_frame_breakdown": { + "frame": worst_work_frame_id, + "world_tick_frame": worst_work_frame_id, + "frame_interval_end_frame": worst_interval_frame_id, + "frame_interval_ms": round(float(worst_interval.get("frame_ms", 0.0)), 3) + if worst_interval else None, + "apply_ms": round(float(worst_apply.get("apply_ms", 0.0)), 3) + if worst_apply else None, + "result_drain_ms": round(float(worst_apply.get("drain_ms", 0.0)), 3) + if worst_apply else None, + "outside_world_tick_ms": round(max(0.0, float(worst_interval.get("frame_ms", 0.0)) + - float(worst_work.get("tick_ms", 0.0))), 3) + if worst_interval else None, + "outside_full_world_tick_ms": round(max(0.0, float(worst_interval.get("frame_ms", 0.0)) + - float(commandlet_tick_by_frame.get(worst_work_frame_id, {}).get("world_tick_ms", 0.0))), 3) + if worst_interval else None, + "full_world_tick_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("world_tick_ms", 0.0)), 3), + "world_pre_actor_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("world_pre_actor_ms", 0.0)), 3), + "world_actor_phase_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("world_actor_phase_ms", 0.0)), 3), + "world_post_actor_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("world_post_actor_ms", 0.0)), 3), + "commandlet_readiness_check_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("readiness_check_ms", 0.0)), 3), + "commandlet_sleep_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("sleep_ms", 0.0)), 3), + "commandlet_loop_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("loop_ms", 0.0)), 3), + "gc_duration_ms": round(float(garbage_collect_by_frame.get( + worst_work_frame_id, {}).get("duration_ms", 0.0)), 3), + "world_tick_ms": round(float(worst_work.get("tick_ms", 0.0)), 3), + "active_generation_workers": int(worst_work.get("active_generation_workers", 0)), + "active_tile_tasks": int(worst_work.get("active_tile_tasks", 0)), + "collision_gate_ms": round(float(worst_work.get("collision_gate_ms", 0.0)), 3), + "update_chunks_ms": round(float(worst_work.get("update_chunks_ms", 0.0)), 3), + "desired_rebuild_ms": round(float(worst_scheduler.get("desired_rebuild_ms", 0.0)), 3), + "pending_drain_ms": round(float(worst_work.get("pending_drain_ms", 0.0)), 3), + "post_apply_callbacks_ms": round(float(worst_work.get("post_apply_callbacks_ms", 0.0)), 3), + "collision_probe_ms": round(float(worst_work.get("collision_probe_ms", 0.0)), 3), + "visibility_reconcile_ms": round(float(worst_work.get("visibility_reconcile_ms", 0.0)), 3), + "visible_apply_probe_ms": round(float(worst_work.get("visible_apply_probe_ms", 0.0)), 3), + "lod0_ready_ms": round(float(worst_work.get("lod0_ready_ms", 0.0)), 3), + "unload_ms": round(float(worst_work.get("unload_ms", 0.0)), 3), + "steady_trace_ms": round(float(worst_work.get("steady_trace_ms", 0.0)), 3), + "modification_ms": round(float(worst_work.get("modification_ms", 0.0)), 3), + "lifecycle_audit_ms": round(float(worst_work.get("lifecycle_audit_ms", 0.0)), 3), + "trace_json_ms": round(float(worst_work.get("trace_json_ms", 0.0)), 3), + "trace_archive_serialize_ms": round(float( + worst_work.get("trace_archive_serialize_ms", 0.0)), 3), + "collision_completion_ms": round(worst_collision_completion_ms, 3), + **{key: round(float(worst_rmc.get(key, 0.0)), 3) + for key in ("component_acquire_ms", "create_section_group_ms", + "create_sections_ms", "collision_handoff_ms")}, + }, + "trace": str(trace.relative_to(ROOT)), "utrace": str(utrace.relative_to(ROOT)), + "abslog": str(abslog.relative_to(ROOT)), "stdout": str(stdout.relative_to(ROOT)), + } + stdout_text = stdout.read_text(encoding="utf-8", errors="replace") + gap_ms, gap_frame = endphysics_gap(stdout) + row["endphysics_to_postphysics_gap_ms_max"] = gap_ms + row["endphysics_gap_frame"] = gap_frame + failure = re.search(r"\[VoxelForgeCoreWorldTest\] failed: ([^\r\n]+)", stdout_text) + if failure: + row["world_test_failure"] = failure.group(1) + with (OUT / "runs.jsonl").open("a", encoding="utf-8") as stream: + stream.write(json.dumps(row, sort_keys=True) + "\n") + (OUT / f"{label}.summary.json").write_text(json.dumps(row, indent=2) + "\n", + encoding="utf-8") + print("[L107UnrealArrival] " + json.dumps(row, sort_keys=True), flush=True) + if exit_code: + raise SystemExit(f"Unreal commandlet failed: {stdout}; exit={exit_code}") + if not full or len(arrival) < 8: + raise SystemExit(f"Unreal arrival trace incomplete: {stdout}; exit={exit_code}") + if insights and (not utrace.is_file() or utrace.stat().st_size < 1024): + raise SystemExit(f"Unreal Insights trace missing or empty: {utrace}") + return row + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--stratum", choices=("FI", "SW"), required=True) + parser.add_argument("--repeat", type=int, default=1) + parser.add_argument("--phase", choices=("probe", "before", "after", "snapshot"), default="probe") + parser.add_argument("--ready-depth", type=int, default=64) + parser.add_argument("--background-cap", type=int, choices=(0, 1), default=1) + parser.add_argument("--worker-reserve", type=int, choices=(0, 1, 2, 3), default=0) + parser.add_argument("--helper-background", type=int, choices=(0, 1), default=1) + parser.add_argument("--tile-background-low", type=int, choices=(0, 1), default=1) + parser.add_argument("--tile-background-normal", type=int, choices=(0, 1), default=0) + parser.add_argument("--startup-boost", type=int, choices=(0, 1), default=1) + parser.add_argument("--tick-logs", action="store_true") + parser.add_argument("--insights", action="store_true") + args = parser.parse_args() + if not EDITOR.is_file(): + raise SystemExit(f"missing Unreal commandlet executable: {EDITOR}") + run(args.stratum, args.repeat, args.phase, args.ready_depth, + args.background_cap, args.worker_reserve, args.helper_background, + args.tile_background_low, args.tile_background_normal, args.startup_boost, + args.tick_logs, args.insights) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux107/patches/startup-overlap-not-merged.patch b/Standalone/Research/WorldAssemblerM3/Linux107/patches/startup-overlap-not-merged.patch new file mode 100644 index 0000000..8df9f23 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux107/patches/startup-overlap-not-merged.patch @@ -0,0 +1,390 @@ +diff --git a/Standalone/Core/src/CoreNamespacePreamble.h b/Standalone/Core/src/CoreNamespacePreamble.h +index 5362f75..df31e40 100644 +--- a/Standalone/Core/src/CoreNamespacePreamble.h ++++ b/Standalone/Core/src/CoreNamespacePreamble.h +@@ -27,6 +27,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/Standalone/Core/src/ShapeWorldInternal.h b/Standalone/Core/src/ShapeWorldInternal.h +index 8626d7e..68889cb 100644 +--- a/Standalone/Core/src/ShapeWorldInternal.h ++++ b/Standalone/Core/src/ShapeWorldInternal.h +@@ -3566,6 +3566,7 @@ public: + double PackedSignBuildSeconds = 0.0; + double PassageSweepPrepSeconds = 0.0; + double CandidateSweepPrepSeconds = 0.0; ++ double StartupPrepOverlapSeconds = 0.0; + double VisualDetailPrepSeconds = 0.0; + double DetailApplySeconds = 0.0; + double SignEdgeDistanceSeconds = 0.0; +@@ -3576,6 +3577,7 @@ public: + double FarParentShapeSeconds = 0.0; + double FarSharedPrimitiveSeconds = 0.0; + double FarBinSeconds = 0.0; ++ double PassageValidationWorkerSeconds = 0.0; + FParallelTiming FarParentGeometryLines; + FParallelTiming RockIntervalLines; + FParallelTiming CandidateAxisLines; +@@ -3810,19 +3812,56 @@ public: + WordsPerZ=(NZ+63u)/64u; + RockWords.assign(static_cast(Size[0])*Size[1]*WordsPerZ,0); + SurfaceDistanceQ=std::make_unique_for_overwrite(static_cast(NodeCount)); +- Substage=Clock::now(); +- BuildPackedRockIntervals(Stats.RockIntervalLines); +- Stats.PackedSignBuildSeconds=std::chrono::duration(Clock::now()-Substage).count(); +- // Native density queries for non-heightfield strates use the exact interval +- // evaluator. Build those native intervals before scanning passage support; +- // the scan itself still completes before any resolved column is composed. +- Substage=Clock::now(); +- BuildPassageVoidCrossingPlan(); +- Stats.PassageSweepPrepSeconds=std::chrono::duration(Clock::now()-Substage).count(); +- Substage=Clock::now(); +- BuildCandidateBands(Stats.CandidateAxisLines,Stats.CandidateMergeLines, +- Stats.CandidateMergeSeconds); +- Stats.CandidateSweepPrepSeconds=std::chrono::duration(Clock::now()-Substage).count(); ++ const bool bOverlapStartupPrep=WorkerCount>=4&&Scheduler; ++ if(bOverlapStartupPrep) ++ { ++ // The packed Z intervals and X/Y candidate sweeps read the same immutable ++ // primitive/bin tables but write separate arrays. Split the helper budget ++ // between them so overlap does not double the configured worker count. ++ const int32 CandidateWorkers=WorkerCount/2; ++ const int32 IntervalWorkers=WorkerCount-CandidateWorkers; ++ const auto PrepStart=Clock::now(); ++ auto PassagePrepFuture=std::async(std::launch::async,[this] ++ { ++ const auto Start=Clock::now(); ++ BuildPassageVoidCrossingPlan(); ++ return std::chrono::duration(Clock::now()-Start).count(); ++ }); ++ auto CandidateAxesFuture=std::async(std::launch::async,[this,CandidateWorkers] ++ { ++ const auto Start=Clock::now(); ++ BuildCandidateAxisBands(Stats.CandidateAxisLines,CandidateWorkers); ++ return std::chrono::duration(Clock::now()-Start).count(); ++ }); ++ Substage=Clock::now(); ++ BuildPackedRockIntervals(Stats.RockIntervalLines,IntervalWorkers); ++ Stats.PackedSignBuildSeconds=std::chrono::duration(Clock::now()-Substage).count(); ++ const double CandidateAxesSeconds=CandidateAxesFuture.get(); ++ Substage=Clock::now(); ++ MergeCandidateBands(Stats.CandidateMergeLines,Stats.CandidateMergeSeconds); ++ const double CandidateMergeWallSeconds= ++ std::chrono::duration(Clock::now()-Substage).count(); ++ Stats.CandidateSweepPrepSeconds=CandidateAxesSeconds+CandidateMergeWallSeconds; ++ Stats.PassageSweepPrepSeconds=PassagePrepFuture.get(); ++ Stats.StartupPrepOverlapSeconds= ++ std::chrono::duration(Clock::now()-PrepStart).count(); ++ } ++ else ++ { ++ Substage=Clock::now(); ++ BuildPackedRockIntervals(Stats.RockIntervalLines); ++ Stats.PackedSignBuildSeconds=std::chrono::duration(Clock::now()-Substage).count(); ++ // Native density queries for non-heightfield strates use the exact interval ++ // evaluator. Build those native intervals before scanning passage support; ++ // the scan itself still completes before any resolved column is composed. ++ Substage=Clock::now(); ++ BuildPassageVoidCrossingPlan(); ++ Stats.PassageSweepPrepSeconds=std::chrono::duration(Clock::now()-Substage).count(); ++ Substage=Clock::now(); ++ BuildCandidateBands(Stats.CandidateAxisLines,Stats.CandidateMergeLines, ++ Stats.CandidateMergeSeconds); ++ Stats.CandidateSweepPrepSeconds=std::chrono::duration(Clock::now()-Substage).count(); ++ } + Substage=Clock::now(); + if(!BuildGpuVisualDetailNodeSets(InGpuVisualDetailTiles,InCpuVisualDetailTiles,Error)) + return false; +@@ -3877,9 +3916,10 @@ public: + Stats.CandidateMergeLines.WallSeconds*1000.0,Stats.CandidateMergeLines.WorkerSeconds*1000.0, + Stats.CandidateMergeSeconds*1000.0,Stats.DetailBandLines.WallSeconds*1000.0, + Stats.DetailBandLines.WorkerSeconds*1000.0); +- std::printf("[ShapeSubstage] detail_apply=%.3fms sign_edge_distances=%.3fms passage_validation=%.3fms shared_field_primitives=%.3fms far_parents=%.3fms\n", ++ std::printf("[ShapeSubstage] detail_apply=%.3fms sign_edge_distances=%.3fms passage_validation=%.3fms passage_validation_worker=%.3fms shared_field_primitives=%.3fms far_parents=%.3fms\n", + Stats.DetailApplySeconds*1000.0,Stats.SignEdgeDistanceSeconds*1000.0, +- Stats.PassageValidationSeconds*1000.0,Stats.SharedFieldSeconds*1000.0, ++ Stats.PassageValidationSeconds*1000.0,Stats.PassageValidationWorkerSeconds*1000.0, ++ Stats.SharedFieldSeconds*1000.0, + Stats.FarShapeSeconds*1000.0); + std::printf("[ShapeBreakdown] owner_shapes=%.3fms owner_shape_worker=%.3fms owner_bins=%.3fms packed_signs=%.3fms passage_prep=%.3fms candidate_sweeps=%.3fms visual_detail_prep=%.3fms detail_band=%.3fms sign_edge_repair=%.3fms route_validation=%.3fms far_parent_shapes=%.3fms far_shared_primitives=%.3fms far_bins=%.3fms\n", + Stats.LocalPrimitiveSeconds*1000.0,Stats.OwnerShapeWorkerSeconds*1000.0, +@@ -3889,6 +3929,10 @@ public: + Stats.DetailApplySeconds*1000.0,Stats.SignEdgeDistanceSeconds*1000.0, + Stats.PassageValidationSeconds*1000.0,Stats.FarParentShapeSeconds*1000.0, + Stats.FarSharedPrimitiveSeconds*1000.0,Stats.FarBinSeconds*1000.0); ++ if(bOverlapStartupPrep) ++ std::printf("[ShapeStartupOverlap] packed_candidate_passage_wall=%.3fms packed_helpers=%d candidate_helpers=%d mode=parallel_exact_disjoint_outputs\n", ++ Stats.StartupPrepOverlapSeconds*1000.0,WorkerCount-WorkerCount/2, ++ WorkerCount/2); + std::printf("[ShapeAllocation] words=%.2fMiB candidate_axis_words=%.2fMiB proxy=%.2fMiB primitives=%.2fMiB spatial_bins=%llu spatial_bin_refs=%llu spatial_bin_capacity=%llu inline_bins=%llu overflow_bins=%llu far_primitive_build=%.3fms far_bin_rebuild=%.3fms\n", + static_cast(WordBytes)/1048576.0,static_cast(CandidateAxisBytes)/1048576.0, + static_cast(Stats.SignNodes*sizeof(int16))/1048576.0, +@@ -12423,7 +12467,7 @@ private: + } + } + +- void BuildPackedRockIntervals(FParallelTiming& Timing) ++ void BuildPackedRockIntervals(FParallelTiming& Timing,int32 ParallelWorkerLimit=0) + { + const int32 NX=Size[0],NY=Size[1],NZ=Size[2]; + CandidateAxisWords[2].assign(static_cast(NX)*NY*WordsPerZ,0); +@@ -12482,11 +12526,10 @@ private: + for(const FInterval& R:Solid)FillBits(RockWords,Base,R.Lo,R.Hi,true); + // Critical passage air is the final authored override, including at landing floors. + for(const FInterval& R:ProtectedAir)FillBits(RockWords,Base,R.Lo,R.Hi,false); +- },&Timing,nullptr,16); ++ },&Timing,nullptr,16,ParallelWorkerLimit); + } + +- void BuildCandidateBands(FParallelTiming& AxisTiming,FParallelTiming& MergeTiming, +- double& MergeSeconds) ++ void BuildCandidateAxisBands(FParallelTiming& AxisTiming,int32 ParallelWorkerLimit=0) + { + const int32 N[3]={Size[0],Size[1],Size[2]}; + for(int32 Axis=0;Axis<3;++Axis) +@@ -12513,8 +12556,12 @@ private: + FillBits(CandidateAxisWords[Axis],Base,Lo-Min[Axis],Hi-Min[Axis],true); + FillBits(CandidateAxisWords[Axis],Base,Lo2-Min[Axis],Hi2-Min[Axis],true); + } +- },&AxisTiming,nullptr,16); ++ },&AxisTiming,nullptr,16,ParallelWorkerLimit); + } ++ } ++ ++ void MergeCandidateBands(FParallelTiming& MergeTiming,double& MergeSeconds) ++ { + const auto MergeStart=std::chrono::steady_clock::now(); + CandidateWords=std::move(CandidateAxisWords[2]); + for(int32 Axis=0;Axis<2;++Axis) +@@ -12563,6 +12610,13 @@ private: + MergeSeconds=std::max(0.0,MergeElapsed-MergeTiming.WallSeconds); + } + ++ void BuildCandidateBands(FParallelTiming& AxisTiming,FParallelTiming& MergeTiming, ++ double& MergeSeconds) ++ { ++ BuildCandidateAxisBands(AxisTiming); ++ MergeCandidateBands(MergeTiming,MergeSeconds); ++ } ++ + bool BuildGpuVisualDetailNodeSets( + const std::vector& GpuTiles, + const std::vector& CpuTiles, +@@ -14134,59 +14188,114 @@ private: + &&ResolvedLayout&&!ResolvedLayout->passages.empty()) + { + constexpr double SampleStep=0.5; +- uint32 PassageIndex=0; +- for(const auto& passage:ResolvedLayout->passages) ++ constexpr double SubvoxelsPerVoxel=static_cast( ++ VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel); ++ struct FRoute ++ { ++ std::string Id; ++ std::size_t PointCount=0; ++ uint64 Samples=0,Failures=0; ++ int32 FirstFailure[3]={0,0,0}; ++ }; ++ struct FSegment ++ { ++ std::size_t Route=0,Index=0,SampleCount=0; ++ double Length=0.0; ++ uint64 Failures=0; ++ }; ++ struct FSample + { +- const auto& points=passage.controlPoints; +- if(points.size()<2){++PassageIndex;continue;} +- uint64 routeSamples=0,routeFailures=0; +- int32 firstFailure[3]={0,0,0}; +- for(std::size_t segment=0;segment+1(a[0])/VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel; +- const double ay=static_cast(a[1])/VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel; +- const double az=static_cast(a[2])/VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel; +- const double bx=static_cast(b[0])/VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel; +- const double by=static_cast(b[1])/VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel; +- const double bz=static_cast(b[2])/VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel; +- const double length=std::sqrt((bx-ax)*(bx-ax)+(by-ay)*(by-ay)+(bz-az)*(bz-az)); +- const int32 count=std::max(1,static_cast(std::ceil(length/SampleStep))); +- uint64 segmentFailures=0; +- for(int32 sample=0;sample<=count;++sample) ++ std::size_t Segment=0; ++ double X=0.0,Y=0.0,Z=0.0; ++ int32 VX=0,VY=0,VZ=0; ++ }; ++ std::vector Routes; ++ std::vector Segments; ++ std::vector Samples; ++ Routes.reserve(ResolvedLayout->passages.size()); ++ for(std::size_t PassageIndex=0; ++ PassageIndexpassages.size();++PassageIndex) ++ { ++ const auto& Passage=ResolvedLayout->passages[PassageIndex]; ++ const auto& Points=Passage.controlPoints; ++ FRoute Route;Route.Id=Passage.stableId;Route.PointCount=Points.size(); ++ const std::size_t RouteIndex=Routes.size(); ++ Routes.push_back(std::move(Route)); ++ if(Points.size()<2)continue; ++ for(std::size_t SegmentIndex=0;SegmentIndex+1(A[0])/SubvoxelsPerVoxel; ++ const double AY=static_cast(A[1])/SubvoxelsPerVoxel; ++ const double AZ=static_cast(A[2])/SubvoxelsPerVoxel; ++ const double BX=static_cast(B[0])/SubvoxelsPerVoxel; ++ const double BY=static_cast(B[1])/SubvoxelsPerVoxel; ++ const double BZ=static_cast(B[2])/SubvoxelsPerVoxel; ++ const double DX=BX-AX,DY=BY-AY,DZ=BZ-AZ; ++ FSegment Segment;Segment.Route=RouteIndex;Segment.Index=SegmentIndex; ++ Segment.Length=std::sqrt(DX*DX+DY*DY+DZ*DZ); ++ const int32 Count=std::max(1,static_cast( ++ std::ceil(Segment.Length/SampleStep))); ++ Segment.SampleCount=static_cast(Count)+1; ++ const std::size_t SegmentIndexInList=Segments.size(); ++ for(int32 SampleIndex=0;SampleIndex<=Count;++SampleIndex) + { +- const double t=static_cast(sample)/count; +- const int32 vx=static_cast(std::lround(ax+(bx-ax)*t)); +- const int32 vy=static_cast(std::lround(ay+(by-ay)*t)); +- const int32 vz=static_cast(std::lround(az+(bz-az)*t)); +- ++Stats.WalkSamples; +- ++routeSamples; +- const float density=DensityAt(static_cast(ax+(bx-ax)*t), +- static_cast(ay+(by-ay)*t), +- static_cast(az+(bz-az)*t)); +- if(!std::isfinite(density)||density<0.0f) +- { +- ++Stats.WalkFailures; +- ++routeFailures; +- ++segmentFailures; +- if(routeFailures==1) +- {firstFailure[0]=vx;firstFailure[1]=vy;firstFailure[2]=vz;} +- } ++ const double T=static_cast(SampleIndex)/Count; ++ FSample Sample;Sample.Segment=SegmentIndexInList; ++ Sample.X=AX+DX*T;Sample.Y=AY+DY*T;Sample.Z=AZ+DZ*T; ++ Sample.VX=static_cast(std::lround(Sample.X)); ++ Sample.VY=static_cast(std::lround(Sample.Y)); ++ Sample.VZ=static_cast(std::lround(Sample.Z)); ++ Samples.push_back(Sample); + } +- if(segmentFailures) +- std::fprintf(stderr,"[WalkabilitySegment] source=resolved_core_layout passage=%s segment=%zu failures=%llu/%llu length=%.2f\n", +- passage.stableId.c_str(),segment, +- static_cast(segmentFailures), +- static_cast(std::max(1,static_cast(std::ceil(length/SampleStep))+1)), +- length); +- } +- if(routeFailures) ++ Segments.push_back(Segment); ++ } ++ } ++ std::vector Failures(Samples.size(),0); ++ FParallelTiming ValidationTiming; ++ ParallelLines(Samples.size(),[&](size_t SampleIndex) ++ { ++ const FSample& Sample=Samples[SampleIndex]; ++ const float Density=DensityAt(static_cast(Sample.X), ++ static_cast(Sample.Y),static_cast(Sample.Z)); ++ Failures[SampleIndex]=!std::isfinite(Density)||Density<0.0f; ++ },&ValidationTiming,nullptr,32); ++ Stats.PassageValidationWorkerSeconds=ValidationTiming.WorkerSeconds; ++ for(const FSegment& Segment:Segments) ++ Routes[Segment.Route].Samples+=Segment.SampleCount; ++ for(std::size_t SampleIndex=0;SampleIndex(Segment.Failures), ++ static_cast(Segment.SampleCount),Segment.Length); ++ } ++ uint32 PassageIndex=0; ++ for(const FRoute& Route:Routes) ++ { ++ if(Route.Failures) + std::fprintf(stderr,"[WalkabilityRoute] source=resolved_core_layout passage=%u id=%s points=%zu failures=%llu/%llu first_solid=(%d,%d,%d)\n", +- PassageIndex,passage.stableId.c_str(),points.size(), +- static_cast(routeFailures), +- static_cast(routeSamples),firstFailure[0], +- firstFailure[1],firstFailure[2]); ++ PassageIndex,Route.Id.c_str(),Route.PointCount, ++ static_cast(Route.Failures), ++ static_cast(Route.Samples),Route.FirstFailure[0], ++ Route.FirstFailure[1],Route.FirstFailure[2]); + ++PassageIndex; + } + if(Stats.WalkFailures) +@@ -14199,10 +14308,8 @@ private: + static_cast(Stats.WalkSamples)); + return Stats.WalkFailures==0; + } +- // An imported Unreal-resolved layout is the canonical route source. An empty +- // passage list is meaningful for blocked controls, so never fall back to the +- // regenerated legacy owner route in that case. Recipe overrides without an +- // exported Core plan retain their generated owner route audit. ++ // An imported Unreal-resolved layout is the canonical route source. An empty ++ // passage list is meaningful for blocked controls; never substitute regenerated routes. + if(Owner&&Owner->bHasUnrealResolvedCorePlan&&ResolvedLayout)return true; + constexpr double SampleStep=0.5; + uint32 PassageIndex=0; +@@ -14283,10 +14390,13 @@ private: + } + + template void ParallelLines(size_t Count,Fn&& Work,FParallelTiming* Timing=nullptr, +- std::array* WorkerActiveSeconds=nullptr,size_t WorkBatch=1) const ++ std::array* WorkerActiveSeconds=nullptr,size_t WorkBatch=1, ++ int32 ParallelWorkerLimit=0) const + { + const auto WallStart=std::chrono::steady_clock::now(); +- if(WorkerCount<=1||Count<512||!Scheduler) ++ const int32 EffectiveWorkerLimit=ParallelWorkerLimit>0 ++ ?std::clamp(ParallelWorkerLimit,1,WorkerCount):WorkerCount; ++ if(EffectiveWorkerLimit<=1||Count<512||!Scheduler) + { + const auto WorkerStart=std::chrono::steady_clock::now(); + for(size_t I=0;I ActiveSeconds{}; + const double WorkerSeconds=Scheduler->parallel_for(Count,WorkBatch, +- static_cast(WorkerCount),&Dispatch,Invoke,ActiveSeconds.data(),ActiveSeconds.size()); ++ static_cast(EffectiveWorkerLimit),&Dispatch,Invoke, ++ ActiveSeconds.data(),ActiveSeconds.size()); + if(WorkerActiveSeconds)*WorkerActiveSeconds=ActiveSeconds; + if(Timing) + { +diff --git a/Standalone/Source/StandaloneShapeWorld.h b/Standalone/Source/StandaloneShapeWorld.h +index 7fa23d4..ae90b35 100644 +--- a/Standalone/Source/StandaloneShapeWorld.h ++++ b/Standalone/Source/StandaloneShapeWorld.h +@@ -1,3 +1,5 @@ + #pragma once + ++#include ++ + #include "../Core/src/ShapeWorldInternal.h"