research: post-swept-tube near-ready cost account (L103, measurement only)
FI/TN LOD0-2 W12 worker CPU 13.9/12.2 s: passages 40/35%, far-shell refinement (HasDownwardHit/ HasUpwardHit ~1.7-1.9 s) in 'other' 27/30%, support 14/13%, dispatch 8%, base field 6%, bands 6%. Workers 28%/24% idle during LOD0-1. Near-ready W12: SW 0.77, FI 1.51, TN 1.38, VS 0.96 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed
1 parent
c38e683292
commit
8d62ba57ed
7 files changed
+973
No files matched your search
@@ -0,0 +1,344 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Close L103 sampler costs against direct grid and tile-worker timings."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l103/census"
|
||||
WORKERS = 12
|
||||
CALLERS = (
|
||||
"unknown", "density_grid", "gradient_normal_fallback", "mc_midpoint_correction",
|
||||
"skirt_surface_vertex", "skirt_extruded_vertex", "skirt_clipped_vertex",
|
||||
"skirt_refinement_face", "skirt_coarse_centroid", "far_shell_surface_query",
|
||||
"far_shell_projection_normal", "far_shell_stratum_refinement",
|
||||
)
|
||||
SKIRT_CALLERS = {
|
||||
"skirt_surface_vertex": "skirt_surface_vertex_offgrid",
|
||||
"skirt_extruded_vertex": "skirt_extruded_vertex_offgrid",
|
||||
"skirt_clipped_vertex": "skirt_clipped_vertex_offgrid",
|
||||
"skirt_refinement_face": "skirt_refinement_face_offgrid",
|
||||
"skirt_coarse_centroid": "skirt_coarse_centroid_offgrid",
|
||||
}
|
||||
ARRAY_FIELDS = (
|
||||
"sweep_missing_by_caller", "sweep_exact_fallback_by_caller",
|
||||
"sweep_bound_skip_by_caller", "sweep_tile_free_skip_by_caller",
|
||||
"sweep_vertical_bound_skip_by_caller", "sweep_negative_reach_skip_by_caller",
|
||||
"sweep_positive_reach_skip_by_caller", "density_offgrid_float_by_caller",
|
||||
"density_offgrid_unclamped_by_caller", "density_offgrid_detail_by_caller",
|
||||
)
|
||||
CPU_FIELDS = (
|
||||
"density_float_points", "density_lattice_points", "density_unclamped_points",
|
||||
"grad_calls", "grad_memo_hits", "grad_fallback_calls", "midpoint_attempts",
|
||||
"midpoint_memo_hits", "skirt_density_probes", "skirt_surface_vertex",
|
||||
"skirt_surface_vertex_offgrid", "skirt_extruded_vertex",
|
||||
"skirt_extruded_vertex_offgrid", "skirt_clipped_vertex",
|
||||
"skirt_clipped_vertex_offgrid", "skirt_refinement_face",
|
||||
"skirt_refinement_face_offgrid", "skirt_coarse_centroid",
|
||||
"skirt_coarse_centroid_offgrid",
|
||||
)
|
||||
COMPOSER_FIELDS = (
|
||||
"queries", "boundary_band_queries", "column_distance_calls", "carve_segment_visits",
|
||||
"support_segment_visits", "support_capsule_checks", "later_bore_checks",
|
||||
"other_capsule_checks", "mouth_support_calls", "mouth_route_segment_checks",
|
||||
"relief_field_calls", "sweep_segments", "sweep_pairs", "sweep_lookups",
|
||||
"sweep_cache_hits", "sweep_missing", "sweep_base_samples", "sweep_detail_samples",
|
||||
"sweep_grid_samples", "sweep_no_influence_samples",
|
||||
)
|
||||
SAMPLE = re.compile(
|
||||
r"^\[L77Sample\] pc=(0x\w+) lod=(\d+) stage=(\S+) route=(\S+) "
|
||||
r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M)
|
||||
SAMPLER = re.compile(r"^\[L77Sampler\] action=stop (.*)$", re.M)
|
||||
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
CPU_LINE = re.compile(r"^\[CpuTileStages\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M)
|
||||
COMPOSER_LINE = re.compile(r"^\[PassageComposerTile\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M)
|
||||
WORKER_LINE = re.compile(r"^\[WorldAssemblerLODWorker\] lod=(\d+) (.*)$", re.M)
|
||||
DETAIL_LINE = re.compile(r"^\[WorldAssemblerLODCostDetail\] lod=(\d+) (.*)$", re.M)
|
||||
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||
|
||||
|
||||
def kv(payload: str) -> dict[str, str]:
|
||||
return dict(FIELDS.findall(payload))
|
||||
|
||||
|
||||
def array_value(text: str | None) -> list[int]:
|
||||
values = [int(value) for value in text.split(",")] if text else []
|
||||
if len(values) != len(CALLERS):
|
||||
raise ValueError(f"expected {len(CALLERS)} caller values, found {len(values)}")
|
||||
return values
|
||||
|
||||
|
||||
def sum_rows(rows: list[dict], fields: tuple[str, ...], lod_max: int | None) -> dict:
|
||||
selected = [row for row in rows if lod_max is None or row["lod"] <= lod_max]
|
||||
result = {field: sum(row[field] for row in selected) for field in fields}
|
||||
for field in ARRAY_FIELDS:
|
||||
result[field] = [sum(row[field][i] for row in selected) for i in range(len(CALLERS))]
|
||||
result["tiles"] = len(selected)
|
||||
return result
|
||||
|
||||
|
||||
def analyze_run(run: dict) -> tuple[dict, list[dict], list[dict], list[dict]]:
|
||||
path = ROOT / run["log"]
|
||||
content = path.read_text(encoding="utf-8", errors="replace")
|
||||
sampler_match = SAMPLER.search(content)
|
||||
stamp_match = STAMP.search(content)
|
||||
pipeline_match = PIPELINE.search(content)
|
||||
if sampler_match is None or stamp_match is None or pipeline_match is None:
|
||||
raise RuntimeError(f"sampler, pipeline, or source stamp missing: {path}")
|
||||
pipeline = kv(pipeline_match.group(1))
|
||||
if int(pipeline.get("workers", 0)) != WORKERS:
|
||||
raise RuntimeError(f"expected W{WORKERS} census run: {path}: {pipeline}")
|
||||
sampler = kv(sampler_match.group(1))
|
||||
if sampler.get("status") != "PASS" or int(sampler.get("lost", "-1")) != 0:
|
||||
raise RuntimeError(f"sampler loss or failure: {path} {sampler}")
|
||||
samples = [
|
||||
{"pc": pc, "lod": int(lod), "stage": stage, "route": route,
|
||||
"phase": phase, "halo": int(halo), "weight": int(weight)}
|
||||
for pc, lod, stage, route, phase, halo, weight in SAMPLE.findall(content)
|
||||
]
|
||||
stored = int(sampler["samples"])
|
||||
total_weight = sum(row["weight"] for row in samples)
|
||||
expanded = int(sampler["expanded_ticks"])
|
||||
if len(samples) != stored or total_weight != expanded:
|
||||
raise RuntimeError(f"sample record closure failed: {path} rows={len(samples)}/{stored} "
|
||||
f"weight={total_weight}/{expanded}")
|
||||
scale_s = float(sampler["profiled_thread_cpu_s"]) / expanded
|
||||
|
||||
cpu_rows, composer_rows, worker_rows, detail_rows = [], [], [], []
|
||||
for lod, tail in CPU_LINE.findall(content):
|
||||
values = kv(tail)
|
||||
row = {"lod": int(lod)}
|
||||
row.update({name: int(values.get(name, 0)) for name in CPU_FIELDS})
|
||||
row.update({name: array_value(values.get(name)) for name in ARRAY_FIELDS})
|
||||
cpu_rows.append(row)
|
||||
for lod, tail in COMPOSER_LINE.findall(content):
|
||||
values = kv(tail)
|
||||
row = {"lod": int(lod)}
|
||||
row.update({name: int(values.get(name, 0)) for name in COMPOSER_FIELDS})
|
||||
composer_rows.append(row)
|
||||
for regex, destination, names in (
|
||||
(WORKER_LINE, worker_rows, ("tiles", "worker_sum_s", "column_eval_exclusive_ms",
|
||||
"mc_exclusive_ms", "ground_refine_exclusive_ms",
|
||||
"cap_refine_exclusive_ms", "skirt_ms", "mesh_assembly_ms")),
|
||||
(DETAIL_LINE, detail_rows, ("tiles", "mc_exclusive_ms", "mc_grid_ms",
|
||||
"mc_cell_class_ms", "mc_gradient_ms", "mc_interpolation_ms",
|
||||
"mc_dedup_stream_ms", "mc_other_ms", "ground_exclusive_ms",
|
||||
"cap_exclusive_ms", "stratum_shell_ms", "connector_shell_ms",
|
||||
"stratum_column_compute_ms", "connector_column_compute_ms")),
|
||||
):
|
||||
for lod, tail in regex.findall(content):
|
||||
values = kv(tail)
|
||||
row = {"lod": int(lod)}
|
||||
for name in names:
|
||||
if name == "tiles":
|
||||
row[name] = int(values.get(name, 0))
|
||||
else:
|
||||
row[name] = float(values.get(name, 0))
|
||||
destination.append(row)
|
||||
if not cpu_rows or not composer_rows or not detail_rows:
|
||||
raise RuntimeError(f"instrumented tile output missing: {path}")
|
||||
|
||||
near_samples = [row for row in samples if row["lod"] <= 2]
|
||||
all_phase = Counter()
|
||||
near_phase = Counter()
|
||||
near_route_phase = Counter()
|
||||
near_stage_phase = Counter()
|
||||
near_lod_stage_phase = Counter()
|
||||
for sample in samples:
|
||||
all_phase[sample["phase"]] += sample["weight"]
|
||||
if sample["lod"] <= 2:
|
||||
near_phase[sample["phase"]] += sample["weight"]
|
||||
near_route_phase[(sample["route"], sample["phase"])] += sample["weight"]
|
||||
near_stage_phase[(sample["stage"], sample["phase"])] += sample["weight"]
|
||||
near_lod_stage_phase[(sample["lod"], sample["stage"], sample["phase"])] += sample["weight"]
|
||||
|
||||
near_cpu = sum_rows(cpu_rows, CPU_FIELDS, 2)
|
||||
all_cpu = sum_rows(cpu_rows, CPU_FIELDS, None)
|
||||
near_composer = {field: sum(row[field] for row in composer_rows if row["lod"] <= 2)
|
||||
for field in COMPOSER_FIELDS}
|
||||
all_composer = {field: sum(row[field] for row in composer_rows) for field in COMPOSER_FIELDS}
|
||||
near_workers = [row for row in worker_rows if row["lod"] <= 2]
|
||||
near_details = [row for row in detail_rows if row["lod"] <= 2]
|
||||
direct_worker_s = sum(row["worker_sum_s"] for row in near_workers)
|
||||
direct_grid_s = sum(row["mc_grid_ms"] for row in near_details) / 1000.0
|
||||
sample_near_cpu_s = sum(near_phase.values()) * scale_s
|
||||
sample_grid_s = near_stage_phase[("grid_fill", "base_field")] * scale_s
|
||||
sample_grid_allphase_s = sum(
|
||||
count for (stage, _phase), count in near_stage_phase.items() if stage == "grid_fill") * scale_s
|
||||
near_total_ticks = sum(near_phase.values())
|
||||
near_phase_cpu_s = {phase: direct_worker_s * ticks / near_total_ticks
|
||||
for phase, ticks in near_phase.items()} if near_total_ticks else {}
|
||||
direct_worker_by_lod = {row["lod"]: row["worker_sum_s"] for row in near_workers}
|
||||
sampled_grid_estimate_s = 0.0
|
||||
for lod, worker_s in direct_worker_by_lod.items():
|
||||
stage_ticks = sum(weight for (sample_lod, stage, _phase), weight
|
||||
in near_lod_stage_phase.items()
|
||||
if sample_lod == lod and stage == "grid_fill")
|
||||
lod_ticks = sum(weight for (sample_lod, _stage, _phase), weight
|
||||
in near_lod_stage_phase.items() if sample_lod == lod)
|
||||
if lod_ticks:
|
||||
sampled_grid_estimate_s += worker_s * stage_ticks / lod_ticks
|
||||
# Compare the sample-derived per-LOD grid-fill share with direct mc_grid_ms.
|
||||
grid_closure_pct = 100.0 * sampled_grid_estimate_s / direct_grid_s if direct_grid_s else 0.0
|
||||
near_direct = {name: sum(row[name] for row in near_details)
|
||||
for name in ("mc_exclusive_ms", "mc_grid_ms", "mc_cell_class_ms",
|
||||
"mc_gradient_ms", "mc_interpolation_ms", "mc_dedup_stream_ms",
|
||||
"mc_other_ms", "ground_exclusive_ms", "cap_exclusive_ms",
|
||||
"stratum_shell_ms", "connector_shell_ms",
|
||||
"stratum_column_compute_ms", "connector_column_compute_ms")}
|
||||
all_direct = {name: sum(row[name] for row in detail_rows)
|
||||
for name in ("stratum_shell_ms", "connector_shell_ms",
|
||||
"stratum_column_compute_ms", "connector_column_compute_ms")}
|
||||
|
||||
skip_fields = ("sweep_exact_fallback_by_caller", "sweep_bound_skip_by_caller",
|
||||
"sweep_tile_free_skip_by_caller", "sweep_vertical_bound_skip_by_caller",
|
||||
"sweep_negative_reach_skip_by_caller", "sweep_positive_reach_skip_by_caller")
|
||||
unresolved = [near_cpu["sweep_missing_by_caller"][i]
|
||||
- sum(near_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))]
|
||||
all_unresolved = [all_cpu["sweep_missing_by_caller"][i]
|
||||
- sum(all_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))]
|
||||
skirt_checks = {
|
||||
caller: near_cpu["density_offgrid_float_by_caller"][CALLERS.index(caller)]
|
||||
== near_cpu[probe]
|
||||
for caller, probe in SKIRT_CALLERS.items()
|
||||
}
|
||||
validation = {
|
||||
"sampler_sample_count_and_expanded_tick_sum": True,
|
||||
"all_sweep_misses_match_composer": sum(all_cpu["sweep_missing_by_caller"])
|
||||
== all_composer["sweep_missing"],
|
||||
"all_misses_have_exact_or_skip_disposition": all(value == 0 for value in all_unresolved),
|
||||
"near_misses_have_exact_or_skip_disposition": all(value == 0 for value in unresolved),
|
||||
"near_midpoint_unique_matches_unclamped_offgrid":
|
||||
near_cpu["midpoint_attempts"] - near_cpu["midpoint_memo_hits"]
|
||||
== near_cpu["density_offgrid_unclamped_by_caller"][CALLERS.index("mc_midpoint_correction")],
|
||||
"near_skirt_callers_match_offgrid_float": all(skirt_checks.values()),
|
||||
"near_skirt_caller_checks": skirt_checks,
|
||||
"grid_fill_closure_diagnostic_recorded": direct_grid_s > 0,
|
||||
}
|
||||
base_samples = near_composer["sweep_base_samples"]
|
||||
detail_samples = near_composer["sweep_detail_samples"]
|
||||
total_sweep_samples = base_samples + detail_samples
|
||||
row = {
|
||||
"stratum": run["stratum"], "replicate": run["replicate"],
|
||||
"workers": WORKERS, "source_stamp": stamp_match.group(1),
|
||||
"exit_code": run["exit_code"],
|
||||
"sampler_samples": stored, "expanded_ticks": expanded,
|
||||
"sampler_lost": int(sampler["lost"]), "timer_overruns": int(sampler["timer_overruns"]),
|
||||
"timer_threads": int(sampler["timer_threads"]),
|
||||
"profiled_thread_cpu_s": float(sampler["profiled_thread_cpu_s"]),
|
||||
"whole_process_cpu_s": float(sampler["whole_process_cpu_s"]),
|
||||
"handler_cpu_ms": float(sampler["handler_cpu_ms"]),
|
||||
"handler_share_pct": float(sampler["handler_share_pct"]),
|
||||
"observed_us_per_tick": float(sampler["observed_us_per_tick"]),
|
||||
"near_lod_samples": len(near_samples),
|
||||
"near_lod_weighted_ticks": sum(near_phase.values()),
|
||||
"near_sampled_worker_cpu_s": sample_near_cpu_s,
|
||||
"near_direct_worker_cpu_s": direct_worker_s,
|
||||
"near_sample_to_direct_worker_pct": 100.0 * sample_near_cpu_s / direct_worker_s if direct_worker_s else 0.0,
|
||||
"near_sampled_grid_fill_s": sample_grid_allphase_s,
|
||||
"near_grid_fill_share_estimate_s": sampled_grid_estimate_s,
|
||||
"near_phase_worker_cpu_s": near_phase_cpu_s,
|
||||
"near_phase_worker_cpu_pct": {phase: 100.0 * ticks / near_total_ticks
|
||||
for phase, ticks in near_phase.items()}
|
||||
if near_total_ticks else {},
|
||||
"near_direct_grid_fill_s": direct_grid_s,
|
||||
"near_grid_fill_closure_pct": grid_closure_pct,
|
||||
"near_grid_fill_base_field_samples_s": sample_grid_s,
|
||||
"near_phase_ticks": dict(near_phase),
|
||||
"near_route_phase_ticks": {f"{r}|{p}": count for (r, p), count in near_route_phase.items()},
|
||||
"near_stage_phase_ticks": {f"{s}|{p}": count for (s, p), count in near_stage_phase.items()},
|
||||
"all_phase_ticks": dict(all_phase),
|
||||
"near_direct_stage_ms": near_direct,
|
||||
"all_direct_far_shell_ms": all_direct,
|
||||
"near_cpu_counts": near_cpu, "all_cpu_counts": all_cpu,
|
||||
"near_composer_counts": near_composer, "all_composer_counts": all_composer,
|
||||
"near_sweep_base_samples": base_samples, "near_sweep_detail_samples": detail_samples,
|
||||
"near_sweep_base_share_pct": 100.0 * base_samples / total_sweep_samples if total_sweep_samples else 0.0,
|
||||
"near_sweep_detail_share_pct": 100.0 * detail_samples / total_sweep_samples if total_sweep_samples else 0.0,
|
||||
"all_sweep_base_samples": all_composer["sweep_base_samples"],
|
||||
"all_sweep_detail_samples": all_composer["sweep_detail_samples"],
|
||||
"validation": validation,
|
||||
"run_log": run["log"],
|
||||
}
|
||||
phase_rows = []
|
||||
near_total = sum(near_phase.values())
|
||||
for phase, ticks in sorted(near_phase.items()):
|
||||
phase_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
||||
"phase": phase, "weighted_ticks": ticks,
|
||||
"sampled_cpu_s": ticks * scale_s,
|
||||
"share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0,
|
||||
"direct_worker_cpu_s_estimate": near_phase_cpu_s.get(phase, 0.0),
|
||||
"share_near_direct_worker_cpu_pct":
|
||||
100 * ticks / near_total if near_total else 0})
|
||||
route_rows = []
|
||||
for (route, phase), ticks in sorted(near_route_phase.items()):
|
||||
route_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
||||
"route": route, "phase": phase, "weighted_ticks": ticks,
|
||||
"sampled_cpu_s": ticks * scale_s,
|
||||
"share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0,
|
||||
"direct_worker_cpu_s_estimate":
|
||||
direct_worker_s * ticks / near_total if near_total else 0.0,
|
||||
"share_near_direct_worker_cpu_pct":
|
||||
100 * ticks / near_total if near_total else 0})
|
||||
caller_rows = []
|
||||
for i, caller in enumerate(CALLERS):
|
||||
caller_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
||||
"caller": caller,
|
||||
**{name.removesuffix("_by_caller"): near_cpu[name][i]
|
||||
for name in ARRAY_FIELDS}})
|
||||
return row, phase_rows, route_rows, caller_rows
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict]) -> None:
|
||||
if rows:
|
||||
with path.open("w", encoding="utf-8", newline="") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
runs = json.loads((OUT / "runs.json").read_text(encoding="utf-8"))
|
||||
details, phase_rows, route_rows, caller_rows = [], [], [], []
|
||||
for run in runs:
|
||||
detail, phases, routes, callers = analyze_run(run)
|
||||
details.append(detail)
|
||||
phase_rows.extend(phases)
|
||||
route_rows.extend(routes)
|
||||
caller_rows.extend(callers)
|
||||
means = []
|
||||
for stratum in ("SW", "FI", "TN", "VS"):
|
||||
subset = [row for row in details if row["stratum"] == stratum]
|
||||
for metric in ("near_sampled_worker_cpu_s", "near_direct_worker_cpu_s",
|
||||
"near_sample_to_direct_worker_pct", "near_sampled_grid_fill_s",
|
||||
"near_grid_fill_share_estimate_s", "near_direct_grid_fill_s",
|
||||
"near_grid_fill_closure_pct",
|
||||
"near_sweep_base_share_pct", "near_sweep_detail_share_pct"):
|
||||
means.append({"stratum": stratum, "metric": metric,
|
||||
"runs": len(subset), "mean": statistics.mean(row[metric] for row in subset),
|
||||
"median": statistics.median(row[metric] for row in subset),
|
||||
"min": min(row[metric] for row in subset),
|
||||
"max": max(row[metric] for row in subset)})
|
||||
write_csv(OUT / "phase-cost.csv", phase_rows)
|
||||
write_csv(OUT / "route-phase-cost.csv", route_rows)
|
||||
write_csv(OUT / "caller-dispositions-near.csv", caller_rows)
|
||||
summary = {"runs": details, "metrics": means,
|
||||
"validation_pass": all(
|
||||
all(value for key, value in row["validation"].items()
|
||||
if key != "near_skirt_caller_checks")
|
||||
and all(row["validation"]["near_skirt_caller_checks"].values())
|
||||
for row in details)}
|
||||
(OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print("[L103CensusSummary] " + json.dumps(
|
||||
{"validation_pass": summary["validation_pass"], "metrics": means}, sort_keys=True), flush=True)
|
||||
return 0 if summary["validation_pass"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attribute FI/TN L77 sample PCs to current-source functions and work areas."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l103/census"
|
||||
BINARY = ROOT / "Saved/Linux/l103/profile-bin/VoxelForgeCoreHeadless-l103-profile-clang"
|
||||
SAMPLE = re.compile(
|
||||
r"^\[L77Sample\] pc=(0x\w+) lod=(\d+) stage=(\S+) route=(\S+) "
|
||||
r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M)
|
||||
TARGETS = {
|
||||
"sweep_base_prepare": "prepareSweptTubeBase",
|
||||
"sweep_lookup": "sweptSample",
|
||||
"exact_route_visit": "VisitNearestPassage",
|
||||
"miss_reach_bound": "PassagePotentialWithinNode",
|
||||
"far_shell_down_hit": "HasDownwardHit",
|
||||
"far_shell_up_hit": "HasUpwardHit",
|
||||
}
|
||||
|
||||
|
||||
def load_samples() -> tuple[list[dict], set[str]]:
|
||||
rows, pcs = [], set()
|
||||
for stratum in ("FI", "TN"):
|
||||
for replicate in (1, 2, 3):
|
||||
log = OUT / f"{stratum.lower()}-r{replicate}.log"
|
||||
content = log.read_text(encoding="utf-8", errors="replace")
|
||||
for pc, lod, stage, route, phase, halo, weight in SAMPLE.findall(content):
|
||||
if int(lod) > 2:
|
||||
continue
|
||||
pcs.add(pc)
|
||||
rows.append({"stratum": stratum, "replicate": replicate, "pc": pc,
|
||||
"lod": int(lod), "stage": stage, "route": route,
|
||||
"phase": phase, "weight": int(weight)})
|
||||
return rows, pcs
|
||||
|
||||
|
||||
def symbolize(pcs: set[str]) -> dict[str, list[dict]]:
|
||||
command = ["llvm-symbolizer-19", "--demangle", "--inlines", "--output-style=JSON",
|
||||
f"--obj={BINARY}"]
|
||||
result = subprocess.run(command, input="\n".join(sorted(pcs)) + "\n",
|
||||
capture_output=True, text=True, check=True)
|
||||
decoder = json.JSONDecoder()
|
||||
offset = 0
|
||||
output = {}
|
||||
while offset < len(result.stdout):
|
||||
while offset < len(result.stdout) and result.stdout[offset].isspace():
|
||||
offset += 1
|
||||
if offset == len(result.stdout):
|
||||
break
|
||||
item, offset = decoder.raw_decode(result.stdout, offset)
|
||||
output[item["Address"]] = item.get("Symbol", [])
|
||||
if output.keys() != pcs:
|
||||
raise RuntimeError(f"symbolizer resolved {len(output)}/{len(pcs)} sampled PCs")
|
||||
return output
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict]) -> None:
|
||||
with path.open("w", encoding="utf-8", newline="") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not BINARY.is_file():
|
||||
raise SystemExit(f"missing profile binary: {BINARY}")
|
||||
samples, pcs = load_samples()
|
||||
symbols = symbolize(pcs)
|
||||
function_totals = Counter()
|
||||
target_totals = Counter()
|
||||
for sample in samples:
|
||||
frames = symbols[sample["pc"]]
|
||||
top_function = frames[0].get("FunctionName", "?") if frames else "?"
|
||||
function_totals[(sample["stratum"], sample["stage"], sample["phase"],
|
||||
top_function)] += sample["weight"]
|
||||
seen = set()
|
||||
for name, token in TARGETS.items():
|
||||
matches = [frame for frame in frames if token in frame.get("FunctionName", "")]
|
||||
if not matches:
|
||||
continue
|
||||
frame = matches[0]
|
||||
line = int(frame.get("Line", 0))
|
||||
category = name
|
||||
if name == "sweep_base_prepare":
|
||||
category = ("sweep_prepare_base" if 14249 <= line <= 14435 else
|
||||
"sweep_prepare_detail" if 14436 <= line <= 14521 else
|
||||
"sweep_prepare_unlocated")
|
||||
key = (sample["stratum"], sample["stage"], sample["phase"], category)
|
||||
if key not in seen:
|
||||
target_totals[key] += sample["weight"]
|
||||
seen.add(key)
|
||||
|
||||
by_function = [
|
||||
{"stratum": stratum, "stage": stage, "phase": phase,
|
||||
"weighted_ticks": ticks, "top_function": function}
|
||||
for (stratum, stage, phase, function), ticks in
|
||||
sorted(function_totals.items(), key=lambda item: (item[0][:3], -item[1]))
|
||||
]
|
||||
by_target = [
|
||||
{"stratum": stratum, "stage": stage, "phase": phase,
|
||||
"work_area": target, "weighted_ticks": ticks}
|
||||
for (stratum, stage, phase, target), ticks in sorted(target_totals.items())
|
||||
]
|
||||
write_csv(OUT / "symbol-function-totals.csv", by_function)
|
||||
write_csv(OUT / "symbol-work-area-totals.csv", by_target)
|
||||
print(f"[L103SymbolAnalysis] samples={len(samples)} unique_pcs={len(pcs)} "
|
||||
f"functions={len(by_function)} work_areas={len(by_target)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,125 @@
|
||||
# Linux103 — current near-ready cost account
|
||||
|
||||
Source stamp: `c5f49a45f51856b857a1ee8877b7d777208e67daf64281caefc7ef84e8006410`.
|
||||
L77 cost rows are medians of three W12 profiles per stratum. Phase CPU is estimated by
|
||||
sample share, calibrated to direct tile-worker CPU. Timer weights include `1 + si_overrun`.
|
||||
|
||||
## W12 cost
|
||||
|
||||
| LOD0–2 worker cost | FI | TN | Measurement |
|
||||
|---|---:|---:|---|
|
||||
| Direct tile-worker CPU | 13.919 s | 12.155 s | Direct worker timers |
|
||||
| Passage phase | 5.538 s (39.8%) | 4.209 s (34.6%) | L77 sample allocation |
|
||||
| Other phase | 3.749 s (26.9%) | 3.681 s (30.4%) | L77 sample allocation; mostly far-shell refinement |
|
||||
| Support | 1.864 s (13.7%) | 1.623 s (13.0%) | L77 sample allocation |
|
||||
| Dispatch lookup | 1.107 s (8.0%) | 1.043 s (8.6%) | L77 sample allocation |
|
||||
| Base field | 0.831 s (6.1%) | 0.808 s (6.5%) | L77 sample allocation |
|
||||
| Composition bands | 0.773 s (5.7%) | 0.781 s (6.3%) | L77 sample allocation |
|
||||
| Graph, caps, edits | 0.084 s | 0.053 s | L77 sample allocation |
|
||||
|
||||
Phase rows partition the worker total. Direct stage timers overlap that account: MC total/grid
|
||||
fill was `9.838 / 9.263 s` FI and `7.827 / 7.340 s` TN; skirt mesh was `0.656 / 0.661 s`.
|
||||
Grid-fill sample estimates closed against direct `mc_grid_ms` at 100.0% FI and 99.8% TN.
|
||||
Whole near-worker sample closure was 99.2% FI and 99.1% TN.
|
||||
|
||||
Passage share is 39.8% FI / 34.6% TN. L99's earlier W8 census reported 34.78% / 28.81%;
|
||||
that comparison crosses source stamps and worker counts. Current near exact-fallback calls
|
||||
(901,055 / 607,012) make off-grid exact work a plausible contributor, but its causal share was
|
||||
not isolated.
|
||||
|
||||
Top route × phase allocations (direct-worker-equivalent seconds):
|
||||
|
||||
| Stratum | Largest route × phase costs |
|
||||
|---|---|
|
||||
| FI | packed-scalar passages 4.533; analytic-scalar passages 0.936; packed support 1.651; packed bands 0.641; dispatch 1.095; unknown/other 3.745 |
|
||||
| TN | packed-scalar passages 3.119; analytic-scalar passages 1.089; packed support 1.394; packed bands 0.671; dispatch 1.035; unknown/other 3.673 |
|
||||
|
||||
Passage sweep work: FI used 3,371,576 base and 347,335 detail samples (90.7% / 9.3%);
|
||||
TN used 3,145,983 and 332,547 (90.4% / 9.6%). Symbolized samples attribute about
|
||||
`0.247 / 0.143 s` FI and `0.261 / 0.169 s` TN to base/detail preparation, `1.010 / 0.739 s`
|
||||
to sweep lookups, and `0.822 / 0.585 s` to exact-route visits plus miss-bound tests. Those
|
||||
subcosts are sampled estimates, not separate direct timers.
|
||||
|
||||
Near LOD0–2 sweep disposition was FI `13,903,288 misses = 901,055 exact fallbacks + 13,002,233
|
||||
skips`; TN `9,086,664 = 607,012 + 8,479,652`. The exact-fallback totals include 255,529 FI and
|
||||
250,311 TN skirt queries. Near far-shell stratum refinement made 410,980 FI and 447,978 TN
|
||||
off-grid queries. Symbolized `HasDownwardHit` / `HasUpwardHit` samples attribute about 1.73 s
|
||||
FI and 1.85 s TN to those hit checks.
|
||||
|
||||
Whole-view off-grid caller counts, per run:
|
||||
|
||||
| Caller | FI | TN |
|
||||
|---|---:|---:|
|
||||
| MC midpoint unique points | 29,101 | 29,864 |
|
||||
| Skirt surface / extruded / clipped vertices | 21,380 / 192,868 / 26,365 | 21,082 / 191,186 / 22,643 |
|
||||
| Skirt refinement face / coarse centroid | 188,382 / 210,056 | 184,353 / 209,077 |
|
||||
| Far-shell surface / projection-normal queries | 539,321 / 192 | 539,321 / 192 |
|
||||
| Far-shell stratum refinement | 1,131,895 | 1,145,781 |
|
||||
|
||||
Across all four strata, all-view sweep misses totaled 63,036,259: 1,908,562 exact fallbacks and
|
||||
61,127,697 skips. Skip reasons were feature bound 59,646,691; tile free 147,272; vertical bound
|
||||
160,272; negative reach 1,158,700; positive reach 14,762. Per-caller miss totals equal exact
|
||||
fallbacks plus skips; unaccounted misses are zero. All 12 W12 profiles had zero lost samples,
|
||||
and their parsed sample counts and expanded-tick sums matched.
|
||||
|
||||
## Critical path and readiness
|
||||
|
||||
In the W12 timeline traces, FI LOD0–1 settled at 1.605 s; TN at 1.218 s. The far-parent build
|
||||
then took 171 ms FI / 114 ms TN. The LOD2 batch started as the far parent completed and drained
|
||||
in 56 / 79 ms. Shape compile was 311 / 200 ms in the traces. Worker-pool busy time was 70.7%
|
||||
FI / 75.0% TN, with 28.3% / 24.3% explicitly idle. The long interval is LOD0–1 tile work;
|
||||
the LOD2 tail is short.
|
||||
|
||||
L80 protocol medians, `near LOD0–2 / whole view` seconds, three interleaved runs per cell:
|
||||
|
||||
| Stratum | W8 | W12 |
|
||||
|---|---:|---:|
|
||||
| SW | 0.764 / 1.914 | 0.772 / 1.786 |
|
||||
| FI | 1.612 / 2.593 | 1.508 / 2.404 |
|
||||
| TN | 1.510 / 2.432 | 1.377 / 2.199 |
|
||||
| VS | 1.107 / 2.061 | 0.958 / 1.822 |
|
||||
|
||||
The W12 whole-view pipeline reported shape / far-shape times of 0.199 / 0.114 s FI and
|
||||
0.215 / 0.114 s TN.
|
||||
|
||||
## Next lever
|
||||
|
||||
No single value-preserving code change was clear from the timers, so L103 ships the following
|
||||
ranked candidates. Savings are inferred CPU scenarios; elapsed-time gains and hash preservation
|
||||
need a changed build to measure.
|
||||
|
||||
1. Reduce passage sweep/carve work. It is 5.54 s FI / 4.21 s TN; a 10% reduction would save
|
||||
about 0.55 / 0.42 worker CPU seconds.
|
||||
2. Reduce far-shell hit checks. The symbolized estimate is 1.73 / 1.85 s; a 25% reduction would
|
||||
save about 0.43 / 0.46 worker CPU seconds. The million-plus whole-view refinement queries make
|
||||
this the best bounded target to investigate.
|
||||
3. Reduce support traversal. It is 1.86 / 1.62 s; a 25% reduction would save about
|
||||
0.47 / 0.41 worker CPU seconds.
|
||||
|
||||
## Hashes and checks
|
||||
|
||||
All 24 whole-view runs matched across Clang/GCC W1/W8/W12. The 20 peer comparisons had identical
|
||||
tile counts and zero density, mesh, payload, or topology mismatches. All 24 hashes also equal
|
||||
the corresponding L101 runs.
|
||||
|
||||
| Stratum | Tiles | Clang W1 whole hash: density / mesh / payload / topology |
|
||||
|---|---:|---|
|
||||
| SW | 2,850 | `8723d8c61348bf31 / fe1c3e410373a343 / bf111bbc45e2f017 / c9c562b100b51f7f` |
|
||||
| FI | 2,249 | `6bb633f756e2b8be / fe930e83b0dc6bf3 / 5298602285b6892c / e306f87ae432a4fc` |
|
||||
| TN | 2,233 | `612b2d11fc6bd946 / c026a40c6127d649 / 0f9036914e28fe46 / 94b36e5f73694eff` |
|
||||
| VS | 2,329 | `99c93f4c7155ffb3 / 488d421c6438f529 / 249b5afc12be4540 / 9c5e22f6dc1e44b8` |
|
||||
|
||||
Fast tier: **PASS**, 274 checks (269 pass, five retained diagnostics), 274.4 s. No production
|
||||
`Source/` files changed, so no Unreal build or arrival trace was required.
|
||||
|
||||
## Failed or excluded runs
|
||||
|
||||
- The first fast-tier launch exited before running checks because its temporary binary symlink
|
||||
was wrong. The corrected run passed.
|
||||
- The first readiness launch had no output directory and started no headless process; all 32
|
||||
subsequent warmup/measured runs completed.
|
||||
- The W8 census and an earlier policy-3 census are retained separately and excluded from the
|
||||
final W12 account. All final W12 census, timeline, readiness, and hash headless runs passed.
|
||||
|
||||
Headless runs used `taskset -c 0-11 prlimit --as=6442450944` and a 1,740-second timeout.
|
||||
Artifacts are under `Saved/Linux/l103/`.
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run current-source W12 L77 route/phase and exact-call censuses."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l103/census"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BINARY = ROOT / "Saved/Linux/l103/profile-bin/VoxelForgeCoreHeadless-l103-profile-clang"
|
||||
CAMERAS = {
|
||||
"SW": (8.0, 16.0, -372.2626953125),
|
||||
"FI": (25.8369140625, 3.19921875, -737.0),
|
||||
"TN": (19.166015625, -14.2001953125, -753.3740234375),
|
||||
"VS": (-20.0625, -5.068359375, -900.353515625),
|
||||
}
|
||||
|
||||
|
||||
def run(stratum: str, replicate: int) -> dict:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
log = OUT / f"{stratum.lower()}-r{replicate}.log"
|
||||
camera = ",".join(f"{value:.12f}" for value in CAMERAS[stratum]) + ",0,0"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s",
|
||||
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||
"--stream-pipeline", "--workers=12", "--clip-radius=3", "--max-clip-level=8",
|
||||
"--max-level=8", f"--camera={camera}",
|
||||
"--profile-lod-work", "--profile-passage-composer", "--linux77-profile",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as stream:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
|
||||
check=False, timeout=1740)
|
||||
row = {"stratum": stratum, "replicate": replicate, "workers": 12,
|
||||
"exit_code": result.returncode,
|
||||
"wall_s": round(time.monotonic() - started, 3),
|
||||
"log": str(log.relative_to(ROOT))}
|
||||
print("[L103CensusRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"census run failed: {log}, exit={result.returncode}")
|
||||
return row
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not BINARY.is_file() or not DATA.is_file():
|
||||
raise SystemExit(f"missing input: {BINARY} or {DATA}")
|
||||
runs = []
|
||||
# Three balanced rounds cover every stratum; reverse order in the middle.
|
||||
orders = (("SW", "FI", "TN", "VS"), ("VS", "TN", "FI", "SW"),
|
||||
("FI", "SW", "VS", "TN"))
|
||||
for replicate, order in enumerate(orders, 1):
|
||||
for stratum in order:
|
||||
runs.append(run(stratum, replicate))
|
||||
(OUT / "runs-partial.json").write_text(
|
||||
json.dumps(runs, indent=2) + "\n", encoding="utf-8")
|
||||
(OUT / "runs.json").write_text(json.dumps(runs, indent=2) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check whole-view tile hashes across current Clang/GCC W1/W8/W12 binaries."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l103/hashes"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BINARIES = {
|
||||
"Clang": ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-clang",
|
||||
"GCC": ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-gcc",
|
||||
}
|
||||
CAMERAS = {
|
||||
"SW": (8.0, 16.0, -372.2626953125),
|
||||
"FI": (25.8369140625, 3.19921875, -737.0),
|
||||
"TN": (19.166015625, -14.2001953125, -753.3740234375),
|
||||
"VS": (-20.0625, -5.068359375, -900.353515625),
|
||||
}
|
||||
TILE_HASH = re.compile(
|
||||
r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
|
||||
r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) "
|
||||
r"topology=(0x[0-9a-f]+)$", re.M)
|
||||
WHOLE_HASH = re.compile(
|
||||
r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) "
|
||||
r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M)
|
||||
FIELDS = ("density", "mesh", "payload", "topology")
|
||||
|
||||
|
||||
def run(stratum: str, compiler: str, workers: int) -> dict:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
log = OUT / f"{stratum.lower()}-{compiler.lower()}-w{workers}.log"
|
||||
camera = ",".join(f"{value:.12f}" for value in CAMERAS[stratum]) + ",0,0"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||
"timeout", "1740s", str(BINARIES[compiler]), str(DATA),
|
||||
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
|
||||
f"--workers={workers}", "--clip-radius=3", "--max-clip-level=8",
|
||||
"--max-level=8", f"--camera={camera}", "--stream-hash-tiles",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as stream:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=stream,
|
||||
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||
output = log.read_text(encoding="utf-8", errors="replace")
|
||||
hashes = {
|
||||
(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))):
|
||||
m.group(5, 6, 7, 8) for m in TILE_HASH.finditer(output)
|
||||
}
|
||||
whole = WHOLE_HASH.search(output)
|
||||
stamp = re.search(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", output, re.M)
|
||||
if (result.returncode or whole is None or stamp is None
|
||||
or len(hashes) != int(whole.group(1))):
|
||||
raise RuntimeError(f"hash run failed or incomplete: exit={result.returncode} {log}")
|
||||
row = {
|
||||
"stratum": stratum, "compiler": compiler, "workers": workers,
|
||||
"source_stamp": stamp.group(1), "exit_code": result.returncode,
|
||||
"wall_s": round(time.monotonic() - started, 3), "tile_count": len(hashes),
|
||||
"whole_hash": list(whole.groups()),
|
||||
"tile_hashes": {"|".join(map(str, key)): value for key, value in hashes.items()},
|
||||
"log": str(log.relative_to(ROOT)),
|
||||
}
|
||||
print("[L103HashRun] " + json.dumps(
|
||||
{key: value for key, value in row.items() if key != "tile_hashes"},
|
||||
sort_keys=True), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def compare(stratum: str, reference: dict, candidate: dict) -> dict:
|
||||
ref = {tuple(map(int, key.split("|"))): value
|
||||
for key, value in reference["tile_hashes"].items()}
|
||||
cand = {tuple(map(int, key.split("|"))): value
|
||||
for key, value in candidate["tile_hashes"].items()}
|
||||
keys = set(ref) | set(cand)
|
||||
mismatches = {
|
||||
field: sum(ref.get(key, (None,) * 4)[index] != cand.get(key, (None,) * 4)[index]
|
||||
for key in keys)
|
||||
for index, field in enumerate(FIELDS)
|
||||
}
|
||||
row = {
|
||||
"stratum": stratum,
|
||||
"reference": f"{reference['compiler']}-w{reference['workers']}",
|
||||
"candidate": f"{candidate['compiler']}-w{candidate['workers']}",
|
||||
"tile_counts_match": reference["tile_count"] == candidate["tile_count"],
|
||||
"whole_hashes_match": reference["whole_hash"] == candidate["whole_hash"],
|
||||
"tile_hashes_match": all(value == 0 for value in mismatches.values()) and ref.keys() == cand.keys(),
|
||||
"component_mismatch_tiles": mismatches,
|
||||
"tiles_reference": reference["tile_count"], "tiles_candidate": candidate["tile_count"],
|
||||
}
|
||||
print("[L103HashComparison] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def main() -> int:
|
||||
for path in (*BINARIES.values(), DATA):
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"missing input: {path}")
|
||||
details, comparisons = [], []
|
||||
for stratum in CAMERAS:
|
||||
group = {(compiler, workers): run(stratum, compiler, workers)
|
||||
for compiler in BINARIES for workers in (1, 8, 12)}
|
||||
reference = group[("Clang", 1)]
|
||||
for key, candidate in group.items():
|
||||
if key != ("Clang", 1):
|
||||
comparisons.append(compare(stratum, reference, candidate))
|
||||
details.extend({key: value for key, value in row.items() if key != "tile_hashes"}
|
||||
for row in group.values())
|
||||
(OUT / "runs-partial.json").write_text(json.dumps(
|
||||
{"runs": details, "comparisons": comparisons}, indent=2) + "\n", encoding="utf-8")
|
||||
summary = {"runs": details, "comparisons": comparisons}
|
||||
(OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
return 0 if all(row["tile_counts_match"] and row["whole_hashes_match"]
|
||||
and row["tile_hashes_match"] for row in comparisons) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""L80-protocol near and whole-view readiness at W8/W12 for all owner strata."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l103/readiness"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BINARY = ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-clang"
|
||||
STRATA = {
|
||||
"SW": {"camera": (8.0, 16.0, -372.2626953125), "near": 1408, "whole": 2850},
|
||||
"FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249},
|
||||
"TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233},
|
||||
"VS": {"camera": (-20.0625, -5.068359375, -900.353515625), "near": 1344, "whole": 2329},
|
||||
}
|
||||
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||
PUBLISHED = re.compile(r"^\[WorldAssemblerNearLODPublished\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||
PIPELINE_READY = re.compile(r"^\[WorldAssemblerNearLODPipelineReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||
CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M)
|
||||
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
|
||||
|
||||
def run(stratum: str, workers: int, phase: str, replicate: int) -> dict:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
suffix = f"-r{replicate}" if phase == "measured" else ""
|
||||
log = OUT / f"{stratum.lower()}-w{workers}-{phase}{suffix}.log"
|
||||
camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s",
|
||||
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||
"--stream-pipeline", f"--workers={workers}", "--clip-radius=3",
|
||||
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
|
||||
"--profile-lod-work", "--stream-scheduler-policy=3",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as stream:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
|
||||
check=False, timeout=1740)
|
||||
content = log.read_text(encoding="utf-8", errors="replace")
|
||||
near, published, pipeline_ready = (NEAR.search(content), PUBLISHED.search(content),
|
||||
PIPELINE_READY.search(content))
|
||||
pipeline, config, stamp = PIPELINE.search(content), CONFIG.search(content), STAMP.search(content)
|
||||
if result.returncode or not all((near, published, pipeline_ready, pipeline, config, stamp)):
|
||||
raise RuntimeError(f"readiness run failed or markers missing: {log}; exit={result.returncode}")
|
||||
fields = dict(FIELDS.findall(pipeline.group(1)))
|
||||
config_fields = dict(FIELDS.findall(config.group(1)))
|
||||
spec = STRATA[stratum]
|
||||
if int(config_fields["workers"]) != workers:
|
||||
raise RuntimeError(f"worker mismatch: {log}")
|
||||
if stamp.group(1) != "c5f49a45f51856b857a1ee8877b7d777208e67daf64281caefc7ef84e8006410":
|
||||
raise RuntimeError(f"source stamp mismatch: {log}: {stamp.group(1)}")
|
||||
for marker in (near, published, pipeline_ready):
|
||||
if (int(marker.group(1)), int(marker.group(2))) != (spec["near"], spec["near"]):
|
||||
raise RuntimeError(f"incomplete near set: {log}: {marker.group(1)}/{marker.group(2)}")
|
||||
whole_done, whole_total = (int(value) for value in fields["whole_tiles"].split("/"))
|
||||
if (whole_done, whole_total) != (spec["whole"], spec["whole"]):
|
||||
raise RuntimeError(f"incomplete whole set: {log}: {whole_done}/{whole_total}")
|
||||
row = {
|
||||
"stratum": stratum, "workers": workers, "phase": phase, "replicate": replicate,
|
||||
"exit_code": result.returncode, "source_stamp": stamp.group(1),
|
||||
"near_tiles": spec["near"], "whole_tiles": spec["whole"],
|
||||
"lod02_ready_s": float(near.group(3)),
|
||||
"lod02_published_s": float(published.group(3)),
|
||||
"lod02_pipeline_ready_s": float(pipeline_ready.group(3)),
|
||||
"whole_view_ready_s": float(fields["whole_view_ready"].removesuffix("s")),
|
||||
"shape_s": float(fields["shape"].removesuffix("s")),
|
||||
"far_shape_s": float(fields["far_shape"].removesuffix("s")),
|
||||
"wall_s": round(time.monotonic() - started, 3),
|
||||
"log": str(log.relative_to(ROOT)),
|
||||
}
|
||||
print("[L103ReadinessRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not DATA.is_file() or not BINARY.is_file():
|
||||
raise SystemExit(f"missing input: {DATA} or {BINARY}")
|
||||
for stratum in ("VS", "TN", "FI", "SW"):
|
||||
for workers in (12, 8):
|
||||
run(stratum, workers, "warmup", 0)
|
||||
schedules = (
|
||||
(("SW", 8), ("SW", 12), ("FI", 8), ("FI", 12), ("TN", 8), ("TN", 12), ("VS", 8), ("VS", 12)),
|
||||
(("VS", 12), ("VS", 8), ("TN", 12), ("TN", 8), ("FI", 12), ("FI", 8), ("SW", 12), ("SW", 8)),
|
||||
(("FI", 8), ("FI", 12), ("SW", 8), ("SW", 12), ("VS", 8), ("VS", 12), ("TN", 8), ("TN", 12)),
|
||||
)
|
||||
rows = []
|
||||
for replicate, schedule in enumerate(schedules, 1):
|
||||
for stratum, workers in schedule:
|
||||
rows.append(run(stratum, workers, "measured", replicate))
|
||||
(OUT / "runs-partial.json").write_text(json.dumps(rows, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
summary = []
|
||||
for stratum in STRATA:
|
||||
for workers in (8, 12):
|
||||
group = [row for row in rows if row["stratum"] == stratum and row["workers"] == workers]
|
||||
if len(group) != 3:
|
||||
raise RuntimeError(f"expected three measured runs: {stratum} W{workers}")
|
||||
summary.append({
|
||||
"stratum": stratum, "workers": workers, "measured_runs": 3,
|
||||
**{metric: [row[metric] for row in group]
|
||||
for metric in ("lod02_ready_s", "lod02_published_s",
|
||||
"lod02_pipeline_ready_s", "whole_view_ready_s")},
|
||||
**{f"{metric}_median": statistics.median(row[metric] for row in group)
|
||||
for metric in ("lod02_ready_s", "lod02_published_s",
|
||||
"lod02_pipeline_ready_s", "whole_view_ready_s")},
|
||||
})
|
||||
(OUT / "summary.json").write_text(json.dumps({"runs": rows, "summary": summary}, indent=2)
|
||||
+ "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture L87 cold critical-path and worker-idle traces on current source."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l103/timeline"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BINARY = ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-clang"
|
||||
STRATA = {
|
||||
"FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249},
|
||||
"TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233},
|
||||
}
|
||||
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||
CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M)
|
||||
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
|
||||
|
||||
def run(stratum: str, workers: int) -> dict:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
log = OUT / f"{stratum.lower()}-w{workers}-direct-measured-r1.log"
|
||||
camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s",
|
||||
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||
"--stream-pipeline", f"--workers={workers}", "--clip-radius=3",
|
||||
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
|
||||
"--profile-lod-work", "--stream-scheduler-policy=3", "--cold-worker-timeline",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as stream:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
|
||||
check=False, timeout=1740)
|
||||
content = log.read_text(encoding="utf-8", errors="replace")
|
||||
near, pipeline, config, stamp = (NEAR.search(content), PIPELINE.search(content),
|
||||
CONFIG.search(content), STAMP.search(content))
|
||||
events = re.findall(r"^\[CpuTileTimeline\] event=(\S+)", content, re.M)
|
||||
cold_events = re.findall(r"^\[ColdTimeline\] event=(\S+) edge=(\S+)", content, re.M)
|
||||
if result.returncode or not all((near, pipeline, config, stamp)) or not events:
|
||||
raise RuntimeError(f"timeline run failed or trace missing: {log}; exit={result.returncode}")
|
||||
if stamp.group(1) != "c5f49a45f51856b857a1ee8877b7d777208e67daf64281caefc7ef84e8006410":
|
||||
raise RuntimeError(f"source stamp mismatch: {log}: {stamp.group(1)}")
|
||||
config_fields = dict(FIELDS.findall(config.group(1)))
|
||||
if int(config_fields["workers"]) != workers:
|
||||
raise RuntimeError(f"worker mismatch: {log}")
|
||||
spec = STRATA[stratum]
|
||||
if (int(near.group(1)), int(near.group(2))) != (spec["near"], spec["near"]):
|
||||
raise RuntimeError(f"incomplete near set: {log}: {near.group(1)}/{near.group(2)}")
|
||||
pipeline_fields = dict(FIELDS.findall(pipeline.group(1)))
|
||||
whole = pipeline_fields["whole_tiles"].split("/")
|
||||
if (int(whole[0]), int(whole[1])) != (spec["whole"], spec["whole"]):
|
||||
raise RuntimeError(f"incomplete whole set: {log}: {pipeline_fields['whole_tiles']}")
|
||||
row = {"stratum": stratum, "workers": workers, "exit_code": result.returncode,
|
||||
"source_stamp": stamp.group(1), "near_tiles": spec["near"],
|
||||
"whole_tiles": spec["whole"], "lod02_ready_s": float(near.group(3)),
|
||||
"whole_view_ready_s": float(pipeline_fields["whole_view_ready"].removesuffix("s")),
|
||||
"timeline_events": len(events), "cold_timeline_marks": len(cold_events),
|
||||
"wall_s": round(time.monotonic() - started, 3),
|
||||
"log": str(log.relative_to(ROOT))}
|
||||
print("[L103TimelineRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def main() -> int:
|
||||
for stratum in STRATA:
|
||||
for workers in (8, 12):
|
||||
run(stratum, workers)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user