From 8d62ba57ed67affefcd31bec067c55a4da1e0edf Mon Sep 17 00:00:00 2001 From: Fr0zka Date: Wed, 28 Oct 2026 19:39:47 +0100 Subject: [PATCH] research: post-swept-tube near-ready cost account (L103, measurement only) FI/TN LOD0-2 W12 worker CPU 13.9/12.2 s: passages 40/35%, far-shell refinement (HasDownwardHit/ HasUpwardHit ~1.7-1.9 s) in 'other' 27/30%, support 14/13%, dispatch 8%, base field 6%, bands 6%. Workers 28%/24% idle during LOD0-1. Near-ready W12: SW 0.77, FI 1.51, TN 1.38, VS 0.96 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1 --- .../Linux103/AnalyzeL103Census.py | 344 ++++++++++++++++++ .../Linux103/AnalyzeL103Symbols.py | 119 ++++++ .../WorldAssemblerM3/Linux103/REPORT.md | 125 +++++++ .../Linux103/RunL103Census.py | 64 ++++ .../Linux103/RunL103Hashes.py | 121 ++++++ .../Linux103/RunL103Readiness.py | 121 ++++++ .../Linux103/RunL103Timeline.py | 79 ++++ 7 files changed, 973 insertions(+) create mode 100644 Standalone/Research/WorldAssemblerM3/Linux103/AnalyzeL103Census.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux103/AnalyzeL103Symbols.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux103/REPORT.md create mode 100644 Standalone/Research/WorldAssemblerM3/Linux103/RunL103Census.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux103/RunL103Hashes.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux103/RunL103Readiness.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux103/RunL103Timeline.py diff --git a/Standalone/Research/WorldAssemblerM3/Linux103/AnalyzeL103Census.py b/Standalone/Research/WorldAssemblerM3/Linux103/AnalyzeL103Census.py new file mode 100644 index 0000000..9f9b662 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux103/AnalyzeL103Census.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Close L103 sampler costs against direct grid and tile-worker timings.""" +from __future__ import annotations + +import csv +import json +import re +import statistics +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l103/census" +WORKERS = 12 +CALLERS = ( + "unknown", "density_grid", "gradient_normal_fallback", "mc_midpoint_correction", + "skirt_surface_vertex", "skirt_extruded_vertex", "skirt_clipped_vertex", + "skirt_refinement_face", "skirt_coarse_centroid", "far_shell_surface_query", + "far_shell_projection_normal", "far_shell_stratum_refinement", +) +SKIRT_CALLERS = { + "skirt_surface_vertex": "skirt_surface_vertex_offgrid", + "skirt_extruded_vertex": "skirt_extruded_vertex_offgrid", + "skirt_clipped_vertex": "skirt_clipped_vertex_offgrid", + "skirt_refinement_face": "skirt_refinement_face_offgrid", + "skirt_coarse_centroid": "skirt_coarse_centroid_offgrid", +} +ARRAY_FIELDS = ( + "sweep_missing_by_caller", "sweep_exact_fallback_by_caller", + "sweep_bound_skip_by_caller", "sweep_tile_free_skip_by_caller", + "sweep_vertical_bound_skip_by_caller", "sweep_negative_reach_skip_by_caller", + "sweep_positive_reach_skip_by_caller", "density_offgrid_float_by_caller", + "density_offgrid_unclamped_by_caller", "density_offgrid_detail_by_caller", +) +CPU_FIELDS = ( + "density_float_points", "density_lattice_points", "density_unclamped_points", + "grad_calls", "grad_memo_hits", "grad_fallback_calls", "midpoint_attempts", + "midpoint_memo_hits", "skirt_density_probes", "skirt_surface_vertex", + "skirt_surface_vertex_offgrid", "skirt_extruded_vertex", + "skirt_extruded_vertex_offgrid", "skirt_clipped_vertex", + "skirt_clipped_vertex_offgrid", "skirt_refinement_face", + "skirt_refinement_face_offgrid", "skirt_coarse_centroid", + "skirt_coarse_centroid_offgrid", +) +COMPOSER_FIELDS = ( + "queries", "boundary_band_queries", "column_distance_calls", "carve_segment_visits", + "support_segment_visits", "support_capsule_checks", "later_bore_checks", + "other_capsule_checks", "mouth_support_calls", "mouth_route_segment_checks", + "relief_field_calls", "sweep_segments", "sweep_pairs", "sweep_lookups", + "sweep_cache_hits", "sweep_missing", "sweep_base_samples", "sweep_detail_samples", + "sweep_grid_samples", "sweep_no_influence_samples", +) +SAMPLE = re.compile( + r"^\[L77Sample\] pc=(0x\w+) lod=(\d+) stage=(\S+) route=(\S+) " + r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M) +SAMPLER = re.compile(r"^\[L77Sampler\] action=stop (.*)$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") +CPU_LINE = re.compile(r"^\[CpuTileStages\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M) +COMPOSER_LINE = re.compile(r"^\[PassageComposerTile\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M) +WORKER_LINE = re.compile(r"^\[WorldAssemblerLODWorker\] lod=(\d+) (.*)$", re.M) +DETAIL_LINE = re.compile(r"^\[WorldAssemblerLODCostDetail\] lod=(\d+) (.*)$", re.M) +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M) + + +def kv(payload: str) -> dict[str, str]: + return dict(FIELDS.findall(payload)) + + +def array_value(text: str | None) -> list[int]: + values = [int(value) for value in text.split(",")] if text else [] + if len(values) != len(CALLERS): + raise ValueError(f"expected {len(CALLERS)} caller values, found {len(values)}") + return values + + +def sum_rows(rows: list[dict], fields: tuple[str, ...], lod_max: int | None) -> dict: + selected = [row for row in rows if lod_max is None or row["lod"] <= lod_max] + result = {field: sum(row[field] for row in selected) for field in fields} + for field in ARRAY_FIELDS: + result[field] = [sum(row[field][i] for row in selected) for i in range(len(CALLERS))] + result["tiles"] = len(selected) + return result + + +def analyze_run(run: dict) -> tuple[dict, list[dict], list[dict], list[dict]]: + path = ROOT / run["log"] + content = path.read_text(encoding="utf-8", errors="replace") + sampler_match = SAMPLER.search(content) + stamp_match = STAMP.search(content) + pipeline_match = PIPELINE.search(content) + if sampler_match is None or stamp_match is None or pipeline_match is None: + raise RuntimeError(f"sampler, pipeline, or source stamp missing: {path}") + pipeline = kv(pipeline_match.group(1)) + if int(pipeline.get("workers", 0)) != WORKERS: + raise RuntimeError(f"expected W{WORKERS} census run: {path}: {pipeline}") + sampler = kv(sampler_match.group(1)) + if sampler.get("status") != "PASS" or int(sampler.get("lost", "-1")) != 0: + raise RuntimeError(f"sampler loss or failure: {path} {sampler}") + samples = [ + {"pc": pc, "lod": int(lod), "stage": stage, "route": route, + "phase": phase, "halo": int(halo), "weight": int(weight)} + for pc, lod, stage, route, phase, halo, weight in SAMPLE.findall(content) + ] + stored = int(sampler["samples"]) + total_weight = sum(row["weight"] for row in samples) + expanded = int(sampler["expanded_ticks"]) + if len(samples) != stored or total_weight != expanded: + raise RuntimeError(f"sample record closure failed: {path} rows={len(samples)}/{stored} " + f"weight={total_weight}/{expanded}") + scale_s = float(sampler["profiled_thread_cpu_s"]) / expanded + + cpu_rows, composer_rows, worker_rows, detail_rows = [], [], [], [] + for lod, tail in CPU_LINE.findall(content): + values = kv(tail) + row = {"lod": int(lod)} + row.update({name: int(values.get(name, 0)) for name in CPU_FIELDS}) + row.update({name: array_value(values.get(name)) for name in ARRAY_FIELDS}) + cpu_rows.append(row) + for lod, tail in COMPOSER_LINE.findall(content): + values = kv(tail) + row = {"lod": int(lod)} + row.update({name: int(values.get(name, 0)) for name in COMPOSER_FIELDS}) + composer_rows.append(row) + for regex, destination, names in ( + (WORKER_LINE, worker_rows, ("tiles", "worker_sum_s", "column_eval_exclusive_ms", + "mc_exclusive_ms", "ground_refine_exclusive_ms", + "cap_refine_exclusive_ms", "skirt_ms", "mesh_assembly_ms")), + (DETAIL_LINE, detail_rows, ("tiles", "mc_exclusive_ms", "mc_grid_ms", + "mc_cell_class_ms", "mc_gradient_ms", "mc_interpolation_ms", + "mc_dedup_stream_ms", "mc_other_ms", "ground_exclusive_ms", + "cap_exclusive_ms", "stratum_shell_ms", "connector_shell_ms", + "stratum_column_compute_ms", "connector_column_compute_ms")), + ): + for lod, tail in regex.findall(content): + values = kv(tail) + row = {"lod": int(lod)} + for name in names: + if name == "tiles": + row[name] = int(values.get(name, 0)) + else: + row[name] = float(values.get(name, 0)) + destination.append(row) + if not cpu_rows or not composer_rows or not detail_rows: + raise RuntimeError(f"instrumented tile output missing: {path}") + + near_samples = [row for row in samples if row["lod"] <= 2] + all_phase = Counter() + near_phase = Counter() + near_route_phase = Counter() + near_stage_phase = Counter() + near_lod_stage_phase = Counter() + for sample in samples: + all_phase[sample["phase"]] += sample["weight"] + if sample["lod"] <= 2: + near_phase[sample["phase"]] += sample["weight"] + near_route_phase[(sample["route"], sample["phase"])] += sample["weight"] + near_stage_phase[(sample["stage"], sample["phase"])] += sample["weight"] + near_lod_stage_phase[(sample["lod"], sample["stage"], sample["phase"])] += sample["weight"] + + near_cpu = sum_rows(cpu_rows, CPU_FIELDS, 2) + all_cpu = sum_rows(cpu_rows, CPU_FIELDS, None) + near_composer = {field: sum(row[field] for row in composer_rows if row["lod"] <= 2) + for field in COMPOSER_FIELDS} + all_composer = {field: sum(row[field] for row in composer_rows) for field in COMPOSER_FIELDS} + near_workers = [row for row in worker_rows if row["lod"] <= 2] + near_details = [row for row in detail_rows if row["lod"] <= 2] + direct_worker_s = sum(row["worker_sum_s"] for row in near_workers) + direct_grid_s = sum(row["mc_grid_ms"] for row in near_details) / 1000.0 + sample_near_cpu_s = sum(near_phase.values()) * scale_s + sample_grid_s = near_stage_phase[("grid_fill", "base_field")] * scale_s + sample_grid_allphase_s = sum( + count for (stage, _phase), count in near_stage_phase.items() if stage == "grid_fill") * scale_s + near_total_ticks = sum(near_phase.values()) + near_phase_cpu_s = {phase: direct_worker_s * ticks / near_total_ticks + for phase, ticks in near_phase.items()} if near_total_ticks else {} + direct_worker_by_lod = {row["lod"]: row["worker_sum_s"] for row in near_workers} + sampled_grid_estimate_s = 0.0 + for lod, worker_s in direct_worker_by_lod.items(): + stage_ticks = sum(weight for (sample_lod, stage, _phase), weight + in near_lod_stage_phase.items() + if sample_lod == lod and stage == "grid_fill") + lod_ticks = sum(weight for (sample_lod, _stage, _phase), weight + in near_lod_stage_phase.items() if sample_lod == lod) + if lod_ticks: + sampled_grid_estimate_s += worker_s * stage_ticks / lod_ticks + # Compare the sample-derived per-LOD grid-fill share with direct mc_grid_ms. + grid_closure_pct = 100.0 * sampled_grid_estimate_s / direct_grid_s if direct_grid_s else 0.0 + near_direct = {name: sum(row[name] for row in near_details) + for name in ("mc_exclusive_ms", "mc_grid_ms", "mc_cell_class_ms", + "mc_gradient_ms", "mc_interpolation_ms", "mc_dedup_stream_ms", + "mc_other_ms", "ground_exclusive_ms", "cap_exclusive_ms", + "stratum_shell_ms", "connector_shell_ms", + "stratum_column_compute_ms", "connector_column_compute_ms")} + all_direct = {name: sum(row[name] for row in detail_rows) + for name in ("stratum_shell_ms", "connector_shell_ms", + "stratum_column_compute_ms", "connector_column_compute_ms")} + + skip_fields = ("sweep_exact_fallback_by_caller", "sweep_bound_skip_by_caller", + "sweep_tile_free_skip_by_caller", "sweep_vertical_bound_skip_by_caller", + "sweep_negative_reach_skip_by_caller", "sweep_positive_reach_skip_by_caller") + unresolved = [near_cpu["sweep_missing_by_caller"][i] + - sum(near_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))] + all_unresolved = [all_cpu["sweep_missing_by_caller"][i] + - sum(all_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))] + skirt_checks = { + caller: near_cpu["density_offgrid_float_by_caller"][CALLERS.index(caller)] + == near_cpu[probe] + for caller, probe in SKIRT_CALLERS.items() + } + validation = { + "sampler_sample_count_and_expanded_tick_sum": True, + "all_sweep_misses_match_composer": sum(all_cpu["sweep_missing_by_caller"]) + == all_composer["sweep_missing"], + "all_misses_have_exact_or_skip_disposition": all(value == 0 for value in all_unresolved), + "near_misses_have_exact_or_skip_disposition": all(value == 0 for value in unresolved), + "near_midpoint_unique_matches_unclamped_offgrid": + near_cpu["midpoint_attempts"] - near_cpu["midpoint_memo_hits"] + == near_cpu["density_offgrid_unclamped_by_caller"][CALLERS.index("mc_midpoint_correction")], + "near_skirt_callers_match_offgrid_float": all(skirt_checks.values()), + "near_skirt_caller_checks": skirt_checks, + "grid_fill_closure_diagnostic_recorded": direct_grid_s > 0, + } + base_samples = near_composer["sweep_base_samples"] + detail_samples = near_composer["sweep_detail_samples"] + total_sweep_samples = base_samples + detail_samples + row = { + "stratum": run["stratum"], "replicate": run["replicate"], + "workers": WORKERS, "source_stamp": stamp_match.group(1), + "exit_code": run["exit_code"], + "sampler_samples": stored, "expanded_ticks": expanded, + "sampler_lost": int(sampler["lost"]), "timer_overruns": int(sampler["timer_overruns"]), + "timer_threads": int(sampler["timer_threads"]), + "profiled_thread_cpu_s": float(sampler["profiled_thread_cpu_s"]), + "whole_process_cpu_s": float(sampler["whole_process_cpu_s"]), + "handler_cpu_ms": float(sampler["handler_cpu_ms"]), + "handler_share_pct": float(sampler["handler_share_pct"]), + "observed_us_per_tick": float(sampler["observed_us_per_tick"]), + "near_lod_samples": len(near_samples), + "near_lod_weighted_ticks": sum(near_phase.values()), + "near_sampled_worker_cpu_s": sample_near_cpu_s, + "near_direct_worker_cpu_s": direct_worker_s, + "near_sample_to_direct_worker_pct": 100.0 * sample_near_cpu_s / direct_worker_s if direct_worker_s else 0.0, + "near_sampled_grid_fill_s": sample_grid_allphase_s, + "near_grid_fill_share_estimate_s": sampled_grid_estimate_s, + "near_phase_worker_cpu_s": near_phase_cpu_s, + "near_phase_worker_cpu_pct": {phase: 100.0 * ticks / near_total_ticks + for phase, ticks in near_phase.items()} + if near_total_ticks else {}, + "near_direct_grid_fill_s": direct_grid_s, + "near_grid_fill_closure_pct": grid_closure_pct, + "near_grid_fill_base_field_samples_s": sample_grid_s, + "near_phase_ticks": dict(near_phase), + "near_route_phase_ticks": {f"{r}|{p}": count for (r, p), count in near_route_phase.items()}, + "near_stage_phase_ticks": {f"{s}|{p}": count for (s, p), count in near_stage_phase.items()}, + "all_phase_ticks": dict(all_phase), + "near_direct_stage_ms": near_direct, + "all_direct_far_shell_ms": all_direct, + "near_cpu_counts": near_cpu, "all_cpu_counts": all_cpu, + "near_composer_counts": near_composer, "all_composer_counts": all_composer, + "near_sweep_base_samples": base_samples, "near_sweep_detail_samples": detail_samples, + "near_sweep_base_share_pct": 100.0 * base_samples / total_sweep_samples if total_sweep_samples else 0.0, + "near_sweep_detail_share_pct": 100.0 * detail_samples / total_sweep_samples if total_sweep_samples else 0.0, + "all_sweep_base_samples": all_composer["sweep_base_samples"], + "all_sweep_detail_samples": all_composer["sweep_detail_samples"], + "validation": validation, + "run_log": run["log"], + } + phase_rows = [] + near_total = sum(near_phase.values()) + for phase, ticks in sorted(near_phase.items()): + phase_rows.append({"stratum": run["stratum"], "replicate": run["replicate"], + "phase": phase, "weighted_ticks": ticks, + "sampled_cpu_s": ticks * scale_s, + "share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0, + "direct_worker_cpu_s_estimate": near_phase_cpu_s.get(phase, 0.0), + "share_near_direct_worker_cpu_pct": + 100 * ticks / near_total if near_total else 0}) + route_rows = [] + for (route, phase), ticks in sorted(near_route_phase.items()): + route_rows.append({"stratum": run["stratum"], "replicate": run["replicate"], + "route": route, "phase": phase, "weighted_ticks": ticks, + "sampled_cpu_s": ticks * scale_s, + "share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0, + "direct_worker_cpu_s_estimate": + direct_worker_s * ticks / near_total if near_total else 0.0, + "share_near_direct_worker_cpu_pct": + 100 * ticks / near_total if near_total else 0}) + caller_rows = [] + for i, caller in enumerate(CALLERS): + caller_rows.append({"stratum": run["stratum"], "replicate": run["replicate"], + "caller": caller, + **{name.removesuffix("_by_caller"): near_cpu[name][i] + for name in ARRAY_FIELDS}}) + return row, phase_rows, route_rows, caller_rows + + +def write_csv(path: Path, rows: list[dict]) -> None: + if rows: + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def main() -> int: + runs = json.loads((OUT / "runs.json").read_text(encoding="utf-8")) + details, phase_rows, route_rows, caller_rows = [], [], [], [] + for run in runs: + detail, phases, routes, callers = analyze_run(run) + details.append(detail) + phase_rows.extend(phases) + route_rows.extend(routes) + caller_rows.extend(callers) + means = [] + for stratum in ("SW", "FI", "TN", "VS"): + subset = [row for row in details if row["stratum"] == stratum] + for metric in ("near_sampled_worker_cpu_s", "near_direct_worker_cpu_s", + "near_sample_to_direct_worker_pct", "near_sampled_grid_fill_s", + "near_grid_fill_share_estimate_s", "near_direct_grid_fill_s", + "near_grid_fill_closure_pct", + "near_sweep_base_share_pct", "near_sweep_detail_share_pct"): + means.append({"stratum": stratum, "metric": metric, + "runs": len(subset), "mean": statistics.mean(row[metric] for row in subset), + "median": statistics.median(row[metric] for row in subset), + "min": min(row[metric] for row in subset), + "max": max(row[metric] for row in subset)}) + write_csv(OUT / "phase-cost.csv", phase_rows) + write_csv(OUT / "route-phase-cost.csv", route_rows) + write_csv(OUT / "caller-dispositions-near.csv", caller_rows) + summary = {"runs": details, "metrics": means, + "validation_pass": all( + all(value for key, value in row["validation"].items() + if key != "near_skirt_caller_checks") + and all(row["validation"]["near_skirt_caller_checks"].values()) + for row in details)} + (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print("[L103CensusSummary] " + json.dumps( + {"validation_pass": summary["validation_pass"], "metrics": means}, sort_keys=True), flush=True) + return 0 if summary["validation_pass"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux103/AnalyzeL103Symbols.py b/Standalone/Research/WorldAssemblerM3/Linux103/AnalyzeL103Symbols.py new file mode 100644 index 0000000..8ea2a69 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux103/AnalyzeL103Symbols.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Attribute FI/TN L77 sample PCs to current-source functions and work areas.""" +from __future__ import annotations + +import csv +import json +import re +import subprocess +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l103/census" +BINARY = ROOT / "Saved/Linux/l103/profile-bin/VoxelForgeCoreHeadless-l103-profile-clang" +SAMPLE = re.compile( + r"^\[L77Sample\] pc=(0x\w+) lod=(\d+) stage=(\S+) route=(\S+) " + r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M) +TARGETS = { + "sweep_base_prepare": "prepareSweptTubeBase", + "sweep_lookup": "sweptSample", + "exact_route_visit": "VisitNearestPassage", + "miss_reach_bound": "PassagePotentialWithinNode", + "far_shell_down_hit": "HasDownwardHit", + "far_shell_up_hit": "HasUpwardHit", +} + + +def load_samples() -> tuple[list[dict], set[str]]: + rows, pcs = [], set() + for stratum in ("FI", "TN"): + for replicate in (1, 2, 3): + log = OUT / f"{stratum.lower()}-r{replicate}.log" + content = log.read_text(encoding="utf-8", errors="replace") + for pc, lod, stage, route, phase, halo, weight in SAMPLE.findall(content): + if int(lod) > 2: + continue + pcs.add(pc) + rows.append({"stratum": stratum, "replicate": replicate, "pc": pc, + "lod": int(lod), "stage": stage, "route": route, + "phase": phase, "weight": int(weight)}) + return rows, pcs + + +def symbolize(pcs: set[str]) -> dict[str, list[dict]]: + command = ["llvm-symbolizer-19", "--demangle", "--inlines", "--output-style=JSON", + f"--obj={BINARY}"] + result = subprocess.run(command, input="\n".join(sorted(pcs)) + "\n", + capture_output=True, text=True, check=True) + decoder = json.JSONDecoder() + offset = 0 + output = {} + while offset < len(result.stdout): + while offset < len(result.stdout) and result.stdout[offset].isspace(): + offset += 1 + if offset == len(result.stdout): + break + item, offset = decoder.raw_decode(result.stdout, offset) + output[item["Address"]] = item.get("Symbol", []) + if output.keys() != pcs: + raise RuntimeError(f"symbolizer resolved {len(output)}/{len(pcs)} sampled PCs") + return output + + +def write_csv(path: Path, rows: list[dict]) -> None: + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def main() -> int: + if not BINARY.is_file(): + raise SystemExit(f"missing profile binary: {BINARY}") + samples, pcs = load_samples() + symbols = symbolize(pcs) + function_totals = Counter() + target_totals = Counter() + for sample in samples: + frames = symbols[sample["pc"]] + top_function = frames[0].get("FunctionName", "?") if frames else "?" + function_totals[(sample["stratum"], sample["stage"], sample["phase"], + top_function)] += sample["weight"] + seen = set() + for name, token in TARGETS.items(): + matches = [frame for frame in frames if token in frame.get("FunctionName", "")] + if not matches: + continue + frame = matches[0] + line = int(frame.get("Line", 0)) + category = name + if name == "sweep_base_prepare": + category = ("sweep_prepare_base" if 14249 <= line <= 14435 else + "sweep_prepare_detail" if 14436 <= line <= 14521 else + "sweep_prepare_unlocated") + key = (sample["stratum"], sample["stage"], sample["phase"], category) + if key not in seen: + target_totals[key] += sample["weight"] + seen.add(key) + + by_function = [ + {"stratum": stratum, "stage": stage, "phase": phase, + "weighted_ticks": ticks, "top_function": function} + for (stratum, stage, phase, function), ticks in + sorted(function_totals.items(), key=lambda item: (item[0][:3], -item[1])) + ] + by_target = [ + {"stratum": stratum, "stage": stage, "phase": phase, + "work_area": target, "weighted_ticks": ticks} + for (stratum, stage, phase, target), ticks in sorted(target_totals.items()) + ] + write_csv(OUT / "symbol-function-totals.csv", by_function) + write_csv(OUT / "symbol-work-area-totals.csv", by_target) + print(f"[L103SymbolAnalysis] samples={len(samples)} unique_pcs={len(pcs)} " + f"functions={len(by_function)} work_areas={len(by_target)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux103/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux103/REPORT.md new file mode 100644 index 0000000..e436b58 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux103/REPORT.md @@ -0,0 +1,125 @@ +# Linux103 — current near-ready cost account + +Source stamp: `c5f49a45f51856b857a1ee8877b7d777208e67daf64281caefc7ef84e8006410`. +L77 cost rows are medians of three W12 profiles per stratum. Phase CPU is estimated by +sample share, calibrated to direct tile-worker CPU. Timer weights include `1 + si_overrun`. + +## W12 cost + +| LOD0–2 worker cost | FI | TN | Measurement | +|---|---:|---:|---| +| Direct tile-worker CPU | 13.919 s | 12.155 s | Direct worker timers | +| Passage phase | 5.538 s (39.8%) | 4.209 s (34.6%) | L77 sample allocation | +| Other phase | 3.749 s (26.9%) | 3.681 s (30.4%) | L77 sample allocation; mostly far-shell refinement | +| Support | 1.864 s (13.7%) | 1.623 s (13.0%) | L77 sample allocation | +| Dispatch lookup | 1.107 s (8.0%) | 1.043 s (8.6%) | L77 sample allocation | +| Base field | 0.831 s (6.1%) | 0.808 s (6.5%) | L77 sample allocation | +| Composition bands | 0.773 s (5.7%) | 0.781 s (6.3%) | L77 sample allocation | +| Graph, caps, edits | 0.084 s | 0.053 s | L77 sample allocation | + +Phase rows partition the worker total. Direct stage timers overlap that account: MC total/grid +fill was `9.838 / 9.263 s` FI and `7.827 / 7.340 s` TN; skirt mesh was `0.656 / 0.661 s`. +Grid-fill sample estimates closed against direct `mc_grid_ms` at 100.0% FI and 99.8% TN. +Whole near-worker sample closure was 99.2% FI and 99.1% TN. + +Passage share is 39.8% FI / 34.6% TN. L99's earlier W8 census reported 34.78% / 28.81%; +that comparison crosses source stamps and worker counts. Current near exact-fallback calls +(901,055 / 607,012) make off-grid exact work a plausible contributor, but its causal share was +not isolated. + +Top route × phase allocations (direct-worker-equivalent seconds): + +| Stratum | Largest route × phase costs | +|---|---| +| FI | packed-scalar passages 4.533; analytic-scalar passages 0.936; packed support 1.651; packed bands 0.641; dispatch 1.095; unknown/other 3.745 | +| TN | packed-scalar passages 3.119; analytic-scalar passages 1.089; packed support 1.394; packed bands 0.671; dispatch 1.035; unknown/other 3.673 | + +Passage sweep work: FI used 3,371,576 base and 347,335 detail samples (90.7% / 9.3%); +TN used 3,145,983 and 332,547 (90.4% / 9.6%). Symbolized samples attribute about +`0.247 / 0.143 s` FI and `0.261 / 0.169 s` TN to base/detail preparation, `1.010 / 0.739 s` +to sweep lookups, and `0.822 / 0.585 s` to exact-route visits plus miss-bound tests. Those +subcosts are sampled estimates, not separate direct timers. + +Near LOD0–2 sweep disposition was FI `13,903,288 misses = 901,055 exact fallbacks + 13,002,233 +skips`; TN `9,086,664 = 607,012 + 8,479,652`. The exact-fallback totals include 255,529 FI and +250,311 TN skirt queries. Near far-shell stratum refinement made 410,980 FI and 447,978 TN +off-grid queries. Symbolized `HasDownwardHit` / `HasUpwardHit` samples attribute about 1.73 s +FI and 1.85 s TN to those hit checks. + +Whole-view off-grid caller counts, per run: + +| Caller | FI | TN | +|---|---:|---:| +| MC midpoint unique points | 29,101 | 29,864 | +| Skirt surface / extruded / clipped vertices | 21,380 / 192,868 / 26,365 | 21,082 / 191,186 / 22,643 | +| Skirt refinement face / coarse centroid | 188,382 / 210,056 | 184,353 / 209,077 | +| Far-shell surface / projection-normal queries | 539,321 / 192 | 539,321 / 192 | +| Far-shell stratum refinement | 1,131,895 | 1,145,781 | + +Across all four strata, all-view sweep misses totaled 63,036,259: 1,908,562 exact fallbacks and +61,127,697 skips. Skip reasons were feature bound 59,646,691; tile free 147,272; vertical bound +160,272; negative reach 1,158,700; positive reach 14,762. Per-caller miss totals equal exact +fallbacks plus skips; unaccounted misses are zero. All 12 W12 profiles had zero lost samples, +and their parsed sample counts and expanded-tick sums matched. + +## Critical path and readiness + +In the W12 timeline traces, FI LOD0–1 settled at 1.605 s; TN at 1.218 s. The far-parent build +then took 171 ms FI / 114 ms TN. The LOD2 batch started as the far parent completed and drained +in 56 / 79 ms. Shape compile was 311 / 200 ms in the traces. Worker-pool busy time was 70.7% +FI / 75.0% TN, with 28.3% / 24.3% explicitly idle. The long interval is LOD0–1 tile work; +the LOD2 tail is short. + +L80 protocol medians, `near LOD0–2 / whole view` seconds, three interleaved runs per cell: + +| Stratum | W8 | W12 | +|---|---:|---:| +| SW | 0.764 / 1.914 | 0.772 / 1.786 | +| FI | 1.612 / 2.593 | 1.508 / 2.404 | +| TN | 1.510 / 2.432 | 1.377 / 2.199 | +| VS | 1.107 / 2.061 | 0.958 / 1.822 | + +The W12 whole-view pipeline reported shape / far-shape times of 0.199 / 0.114 s FI and +0.215 / 0.114 s TN. + +## Next lever + +No single value-preserving code change was clear from the timers, so L103 ships the following +ranked candidates. Savings are inferred CPU scenarios; elapsed-time gains and hash preservation +need a changed build to measure. + +1. Reduce passage sweep/carve work. It is 5.54 s FI / 4.21 s TN; a 10% reduction would save + about 0.55 / 0.42 worker CPU seconds. +2. Reduce far-shell hit checks. The symbolized estimate is 1.73 / 1.85 s; a 25% reduction would + save about 0.43 / 0.46 worker CPU seconds. The million-plus whole-view refinement queries make + this the best bounded target to investigate. +3. Reduce support traversal. It is 1.86 / 1.62 s; a 25% reduction would save about + 0.47 / 0.41 worker CPU seconds. + +## Hashes and checks + +All 24 whole-view runs matched across Clang/GCC W1/W8/W12. The 20 peer comparisons had identical +tile counts and zero density, mesh, payload, or topology mismatches. All 24 hashes also equal +the corresponding L101 runs. + +| Stratum | Tiles | Clang W1 whole hash: density / mesh / payload / topology | +|---|---:|---| +| SW | 2,850 | `8723d8c61348bf31 / fe1c3e410373a343 / bf111bbc45e2f017 / c9c562b100b51f7f` | +| FI | 2,249 | `6bb633f756e2b8be / fe930e83b0dc6bf3 / 5298602285b6892c / e306f87ae432a4fc` | +| TN | 2,233 | `612b2d11fc6bd946 / c026a40c6127d649 / 0f9036914e28fe46 / 94b36e5f73694eff` | +| VS | 2,329 | `99c93f4c7155ffb3 / 488d421c6438f529 / 249b5afc12be4540 / 9c5e22f6dc1e44b8` | + +Fast tier: **PASS**, 274 checks (269 pass, five retained diagnostics), 274.4 s. No production +`Source/` files changed, so no Unreal build or arrival trace was required. + +## Failed or excluded runs + +- The first fast-tier launch exited before running checks because its temporary binary symlink + was wrong. The corrected run passed. +- The first readiness launch had no output directory and started no headless process; all 32 + subsequent warmup/measured runs completed. +- The W8 census and an earlier policy-3 census are retained separately and excluded from the + final W12 account. All final W12 census, timeline, readiness, and hash headless runs passed. + +Headless runs used `taskset -c 0-11 prlimit --as=6442450944` and a 1,740-second timeout. +Artifacts are under `Saved/Linux/l103/`. diff --git a/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Census.py b/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Census.py new file mode 100644 index 0000000..32ababd --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Census.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Run current-source W12 L77 route/phase and exact-call censuses.""" +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l103/census" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l103/profile-bin/VoxelForgeCoreHeadless-l103-profile-clang" +CAMERAS = { + "SW": (8.0, 16.0, -372.2626953125), + "FI": (25.8369140625, 3.19921875, -737.0), + "TN": (19.166015625, -14.2001953125, -753.3740234375), + "VS": (-20.0625, -5.068359375, -900.353515625), +} + + +def run(stratum: str, replicate: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + log = OUT / f"{stratum.lower()}-r{replicate}.log" + camera = ",".join(f"{value:.12f}" for value in CAMERAS[stratum]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark", + "--stream-pipeline", "--workers=12", "--clip-radius=3", "--max-clip-level=8", + "--max-level=8", f"--camera={camera}", + "--profile-lod-work", "--profile-passage-composer", "--linux77-profile", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, + check=False, timeout=1740) + row = {"stratum": stratum, "replicate": replicate, "workers": 12, + "exit_code": result.returncode, + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT))} + print("[L103CensusRun] " + json.dumps(row, sort_keys=True), flush=True) + if result.returncode: + raise RuntimeError(f"census run failed: {log}, exit={result.returncode}") + return row + + +def main() -> int: + if not BINARY.is_file() or not DATA.is_file(): + raise SystemExit(f"missing input: {BINARY} or {DATA}") + runs = [] + # Three balanced rounds cover every stratum; reverse order in the middle. + orders = (("SW", "FI", "TN", "VS"), ("VS", "TN", "FI", "SW"), + ("FI", "SW", "VS", "TN")) + for replicate, order in enumerate(orders, 1): + for stratum in order: + runs.append(run(stratum, replicate)) + (OUT / "runs-partial.json").write_text( + json.dumps(runs, indent=2) + "\n", encoding="utf-8") + (OUT / "runs.json").write_text(json.dumps(runs, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Hashes.py b/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Hashes.py new file mode 100644 index 0000000..9767364 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Hashes.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Check whole-view tile hashes across current Clang/GCC W1/W8/W12 binaries.""" +from __future__ import annotations + +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l103/hashes" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARIES = { + "Clang": ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-clang", + "GCC": ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-gcc", +} +CAMERAS = { + "SW": (8.0, 16.0, -372.2626953125), + "FI": (25.8369140625, 3.19921875, -737.0), + "TN": (19.166015625, -14.2001953125, -753.3740234375), + "VS": (-20.0625, -5.068359375, -900.353515625), +} +TILE_HASH = re.compile( + r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) " + r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) " + r"topology=(0x[0-9a-f]+)$", re.M) +WHOLE_HASH = re.compile( + r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) " + r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M) +FIELDS = ("density", "mesh", "payload", "topology") + + +def run(stratum: str, compiler: str, workers: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + log = OUT / f"{stratum.lower()}-{compiler.lower()}-w{workers}.log" + camera = ",".join(f"{value:.12f}" for value in CAMERAS[stratum]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", + "timeout", "1740s", str(BINARIES[compiler]), str(DATA), + "--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline", + f"--workers={workers}", "--clip-radius=3", "--max-clip-level=8", + "--max-level=8", f"--camera={camera}", "--stream-hash-tiles", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + output = log.read_text(encoding="utf-8", errors="replace") + hashes = { + (int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))): + m.group(5, 6, 7, 8) for m in TILE_HASH.finditer(output) + } + whole = WHOLE_HASH.search(output) + stamp = re.search(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", output, re.M) + if (result.returncode or whole is None or stamp is None + or len(hashes) != int(whole.group(1))): + raise RuntimeError(f"hash run failed or incomplete: exit={result.returncode} {log}") + row = { + "stratum": stratum, "compiler": compiler, "workers": workers, + "source_stamp": stamp.group(1), "exit_code": result.returncode, + "wall_s": round(time.monotonic() - started, 3), "tile_count": len(hashes), + "whole_hash": list(whole.groups()), + "tile_hashes": {"|".join(map(str, key)): value for key, value in hashes.items()}, + "log": str(log.relative_to(ROOT)), + } + print("[L103HashRun] " + json.dumps( + {key: value for key, value in row.items() if key != "tile_hashes"}, + sort_keys=True), flush=True) + return row + + +def compare(stratum: str, reference: dict, candidate: dict) -> dict: + ref = {tuple(map(int, key.split("|"))): value + for key, value in reference["tile_hashes"].items()} + cand = {tuple(map(int, key.split("|"))): value + for key, value in candidate["tile_hashes"].items()} + keys = set(ref) | set(cand) + mismatches = { + field: sum(ref.get(key, (None,) * 4)[index] != cand.get(key, (None,) * 4)[index] + for key in keys) + for index, field in enumerate(FIELDS) + } + row = { + "stratum": stratum, + "reference": f"{reference['compiler']}-w{reference['workers']}", + "candidate": f"{candidate['compiler']}-w{candidate['workers']}", + "tile_counts_match": reference["tile_count"] == candidate["tile_count"], + "whole_hashes_match": reference["whole_hash"] == candidate["whole_hash"], + "tile_hashes_match": all(value == 0 for value in mismatches.values()) and ref.keys() == cand.keys(), + "component_mismatch_tiles": mismatches, + "tiles_reference": reference["tile_count"], "tiles_candidate": candidate["tile_count"], + } + print("[L103HashComparison] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + for path in (*BINARIES.values(), DATA): + if not path.is_file(): + raise SystemExit(f"missing input: {path}") + details, comparisons = [], [] + for stratum in CAMERAS: + group = {(compiler, workers): run(stratum, compiler, workers) + for compiler in BINARIES for workers in (1, 8, 12)} + reference = group[("Clang", 1)] + for key, candidate in group.items(): + if key != ("Clang", 1): + comparisons.append(compare(stratum, reference, candidate)) + details.extend({key: value for key, value in row.items() if key != "tile_hashes"} + for row in group.values()) + (OUT / "runs-partial.json").write_text(json.dumps( + {"runs": details, "comparisons": comparisons}, indent=2) + "\n", encoding="utf-8") + summary = {"runs": details, "comparisons": comparisons} + (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + return 0 if all(row["tile_counts_match"] and row["whole_hashes_match"] + and row["tile_hashes_match"] for row in comparisons) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Readiness.py b/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Readiness.py new file mode 100644 index 0000000..b827d58 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Readiness.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""L80-protocol near and whole-view readiness at W8/W12 for all owner strata.""" +from __future__ import annotations + +import json +import re +import statistics +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l103/readiness" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-clang" +STRATA = { + "SW": {"camera": (8.0, 16.0, -372.2626953125), "near": 1408, "whole": 2850}, + "FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249}, + "TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233}, + "VS": {"camera": (-20.0625, -5.068359375, -900.353515625), "near": 1344, "whole": 2329}, +} +NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +PUBLISHED = re.compile(r"^\[WorldAssemblerNearLODPublished\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +PIPELINE_READY = re.compile(r"^\[WorldAssemblerNearLODPipelineReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") + + +def run(stratum: str, workers: int, phase: str, replicate: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + suffix = f"-r{replicate}" if phase == "measured" else "" + log = OUT / f"{stratum.lower()}-w{workers}-{phase}{suffix}.log" + camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark", + "--stream-pipeline", f"--workers={workers}", "--clip-radius=3", + "--max-clip-level=8", "--max-level=8", f"--camera={camera}", + "--profile-lod-work", "--stream-scheduler-policy=3", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, + check=False, timeout=1740) + content = log.read_text(encoding="utf-8", errors="replace") + near, published, pipeline_ready = (NEAR.search(content), PUBLISHED.search(content), + PIPELINE_READY.search(content)) + pipeline, config, stamp = PIPELINE.search(content), CONFIG.search(content), STAMP.search(content) + if result.returncode or not all((near, published, pipeline_ready, pipeline, config, stamp)): + raise RuntimeError(f"readiness run failed or markers missing: {log}; exit={result.returncode}") + fields = dict(FIELDS.findall(pipeline.group(1))) + config_fields = dict(FIELDS.findall(config.group(1))) + spec = STRATA[stratum] + if int(config_fields["workers"]) != workers: + raise RuntimeError(f"worker mismatch: {log}") + if stamp.group(1) != "c5f49a45f51856b857a1ee8877b7d777208e67daf64281caefc7ef84e8006410": + raise RuntimeError(f"source stamp mismatch: {log}: {stamp.group(1)}") + for marker in (near, published, pipeline_ready): + if (int(marker.group(1)), int(marker.group(2))) != (spec["near"], spec["near"]): + raise RuntimeError(f"incomplete near set: {log}: {marker.group(1)}/{marker.group(2)}") + whole_done, whole_total = (int(value) for value in fields["whole_tiles"].split("/")) + if (whole_done, whole_total) != (spec["whole"], spec["whole"]): + raise RuntimeError(f"incomplete whole set: {log}: {whole_done}/{whole_total}") + row = { + "stratum": stratum, "workers": workers, "phase": phase, "replicate": replicate, + "exit_code": result.returncode, "source_stamp": stamp.group(1), + "near_tiles": spec["near"], "whole_tiles": spec["whole"], + "lod02_ready_s": float(near.group(3)), + "lod02_published_s": float(published.group(3)), + "lod02_pipeline_ready_s": float(pipeline_ready.group(3)), + "whole_view_ready_s": float(fields["whole_view_ready"].removesuffix("s")), + "shape_s": float(fields["shape"].removesuffix("s")), + "far_shape_s": float(fields["far_shape"].removesuffix("s")), + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT)), + } + print("[L103ReadinessRun] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + if not DATA.is_file() or not BINARY.is_file(): + raise SystemExit(f"missing input: {DATA} or {BINARY}") + for stratum in ("VS", "TN", "FI", "SW"): + for workers in (12, 8): + run(stratum, workers, "warmup", 0) + schedules = ( + (("SW", 8), ("SW", 12), ("FI", 8), ("FI", 12), ("TN", 8), ("TN", 12), ("VS", 8), ("VS", 12)), + (("VS", 12), ("VS", 8), ("TN", 12), ("TN", 8), ("FI", 12), ("FI", 8), ("SW", 12), ("SW", 8)), + (("FI", 8), ("FI", 12), ("SW", 8), ("SW", 12), ("VS", 8), ("VS", 12), ("TN", 8), ("TN", 12)), + ) + rows = [] + for replicate, schedule in enumerate(schedules, 1): + for stratum, workers in schedule: + rows.append(run(stratum, workers, "measured", replicate)) + (OUT / "runs-partial.json").write_text(json.dumps(rows, indent=2) + "\n", + encoding="utf-8") + summary = [] + for stratum in STRATA: + for workers in (8, 12): + group = [row for row in rows if row["stratum"] == stratum and row["workers"] == workers] + if len(group) != 3: + raise RuntimeError(f"expected three measured runs: {stratum} W{workers}") + summary.append({ + "stratum": stratum, "workers": workers, "measured_runs": 3, + **{metric: [row[metric] for row in group] + for metric in ("lod02_ready_s", "lod02_published_s", + "lod02_pipeline_ready_s", "whole_view_ready_s")}, + **{f"{metric}_median": statistics.median(row[metric] for row in group) + for metric in ("lod02_ready_s", "lod02_published_s", + "lod02_pipeline_ready_s", "whole_view_ready_s")}, + }) + (OUT / "summary.json").write_text(json.dumps({"runs": rows, "summary": summary}, indent=2) + + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Timeline.py b/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Timeline.py new file mode 100644 index 0000000..5fc37f9 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux103/RunL103Timeline.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Capture L87 cold critical-path and worker-idle traces on current source.""" +from __future__ import annotations + +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l103/timeline" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-clang" +STRATA = { + "FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249}, + "TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233}, +} +NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") + + +def run(stratum: str, workers: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + log = OUT / f"{stratum.lower()}-w{workers}-direct-measured-r1.log" + camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark", + "--stream-pipeline", f"--workers={workers}", "--clip-radius=3", + "--max-clip-level=8", "--max-level=8", f"--camera={camera}", + "--profile-lod-work", "--stream-scheduler-policy=3", "--cold-worker-timeline", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, + check=False, timeout=1740) + content = log.read_text(encoding="utf-8", errors="replace") + near, pipeline, config, stamp = (NEAR.search(content), PIPELINE.search(content), + CONFIG.search(content), STAMP.search(content)) + events = re.findall(r"^\[CpuTileTimeline\] event=(\S+)", content, re.M) + cold_events = re.findall(r"^\[ColdTimeline\] event=(\S+) edge=(\S+)", content, re.M) + if result.returncode or not all((near, pipeline, config, stamp)) or not events: + raise RuntimeError(f"timeline run failed or trace missing: {log}; exit={result.returncode}") + if stamp.group(1) != "c5f49a45f51856b857a1ee8877b7d777208e67daf64281caefc7ef84e8006410": + raise RuntimeError(f"source stamp mismatch: {log}: {stamp.group(1)}") + config_fields = dict(FIELDS.findall(config.group(1))) + if int(config_fields["workers"]) != workers: + raise RuntimeError(f"worker mismatch: {log}") + spec = STRATA[stratum] + if (int(near.group(1)), int(near.group(2))) != (spec["near"], spec["near"]): + raise RuntimeError(f"incomplete near set: {log}: {near.group(1)}/{near.group(2)}") + pipeline_fields = dict(FIELDS.findall(pipeline.group(1))) + whole = pipeline_fields["whole_tiles"].split("/") + if (int(whole[0]), int(whole[1])) != (spec["whole"], spec["whole"]): + raise RuntimeError(f"incomplete whole set: {log}: {pipeline_fields['whole_tiles']}") + row = {"stratum": stratum, "workers": workers, "exit_code": result.returncode, + "source_stamp": stamp.group(1), "near_tiles": spec["near"], + "whole_tiles": spec["whole"], "lod02_ready_s": float(near.group(3)), + "whole_view_ready_s": float(pipeline_fields["whole_view_ready"].removesuffix("s")), + "timeline_events": len(events), "cold_timeline_marks": len(cold_events), + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT))} + print("[L103TimelineRun] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + for stratum in STRATA: + for workers in (8, 12): + run(stratum, workers) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())