Fr0zkaandClaude Opus 5.5
6a80cb0ab3
Near tiles legitimately need floor/ceiling boundary patches, but every hit query scanned the tile's whole XY column. Direction-specific 3D cell bins conservatively index triangle bounds; the existing predicates still decide every hit. Candidate visits per query -88..-99%; identical query and hit counts. Hit-function cost ~1.7-1.9 -> ~0.35 s per FI/TN run. Near-ready @W12: FI 1.63 -> 1.52, TN 1.51 -> 1.38, VS 1.00 -> 0.92 s. All tile/whole hashes equal L103 (Clang/GCC W1/W8/W12). The 24-28% "idle" is in the serial base-shape (~0.24 s) and far-parent (~0.13 s) phases; the LOD0-1 drain is >= 99.4% busy. Fast tier PASS; Linux UE builds; FI Unreal arrival 1.96-1.98 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
345 lines
19 KiB
Python
345 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Close L103 sampler costs against direct grid and tile-worker timings."""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
import re
|
|
import statistics
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[4]
|
|
OUT = ROOT / "Saved/Linux/l104/census"
|
|
WORKERS = 12
|
|
CALLERS = (
|
|
"unknown", "density_grid", "gradient_normal_fallback", "mc_midpoint_correction",
|
|
"skirt_surface_vertex", "skirt_extruded_vertex", "skirt_clipped_vertex",
|
|
"skirt_refinement_face", "skirt_coarse_centroid", "far_shell_surface_query",
|
|
"far_shell_projection_normal", "far_shell_stratum_refinement",
|
|
)
|
|
SKIRT_CALLERS = {
|
|
"skirt_surface_vertex": "skirt_surface_vertex_offgrid",
|
|
"skirt_extruded_vertex": "skirt_extruded_vertex_offgrid",
|
|
"skirt_clipped_vertex": "skirt_clipped_vertex_offgrid",
|
|
"skirt_refinement_face": "skirt_refinement_face_offgrid",
|
|
"skirt_coarse_centroid": "skirt_coarse_centroid_offgrid",
|
|
}
|
|
ARRAY_FIELDS = (
|
|
"sweep_missing_by_caller", "sweep_exact_fallback_by_caller",
|
|
"sweep_bound_skip_by_caller", "sweep_tile_free_skip_by_caller",
|
|
"sweep_vertical_bound_skip_by_caller", "sweep_negative_reach_skip_by_caller",
|
|
"sweep_positive_reach_skip_by_caller", "density_offgrid_float_by_caller",
|
|
"density_offgrid_unclamped_by_caller", "density_offgrid_detail_by_caller",
|
|
)
|
|
CPU_FIELDS = (
|
|
"density_float_points", "density_lattice_points", "density_unclamped_points",
|
|
"grad_calls", "grad_memo_hits", "grad_fallback_calls", "midpoint_attempts",
|
|
"midpoint_memo_hits", "skirt_density_probes", "skirt_surface_vertex",
|
|
"skirt_surface_vertex_offgrid", "skirt_extruded_vertex",
|
|
"skirt_extruded_vertex_offgrid", "skirt_clipped_vertex",
|
|
"skirt_clipped_vertex_offgrid", "skirt_refinement_face",
|
|
"skirt_refinement_face_offgrid", "skirt_coarse_centroid",
|
|
"skirt_coarse_centroid_offgrid",
|
|
)
|
|
COMPOSER_FIELDS = (
|
|
"queries", "boundary_band_queries", "column_distance_calls", "carve_segment_visits",
|
|
"support_segment_visits", "support_capsule_checks", "later_bore_checks",
|
|
"other_capsule_checks", "mouth_support_calls", "mouth_route_segment_checks",
|
|
"relief_field_calls", "sweep_segments", "sweep_pairs", "sweep_lookups",
|
|
"sweep_cache_hits", "sweep_missing", "sweep_base_samples", "sweep_detail_samples",
|
|
"sweep_grid_samples", "sweep_no_influence_samples",
|
|
)
|
|
SAMPLE = re.compile(
|
|
r"^\[L77Sample\] pc=(0x\w+) lod=(\d+) stage=(\S+) route=(\S+) "
|
|
r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M)
|
|
SAMPLER = re.compile(r"^\[L77Sampler\] action=stop (.*)$", re.M)
|
|
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
|
CPU_LINE = re.compile(r"^\[CpuTileStages\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M)
|
|
COMPOSER_LINE = re.compile(r"^\[PassageComposerTile\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M)
|
|
WORKER_LINE = re.compile(r"^\[WorldAssemblerLODWorker\] lod=(\d+) (.*)$", re.M)
|
|
DETAIL_LINE = re.compile(r"^\[WorldAssemblerLODCostDetail\] lod=(\d+) (.*)$", re.M)
|
|
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
|
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
|
|
|
|
|
def kv(payload: str) -> dict[str, str]:
|
|
return dict(FIELDS.findall(payload))
|
|
|
|
|
|
def array_value(text: str | None) -> list[int]:
|
|
values = [int(value) for value in text.split(",")] if text else []
|
|
if len(values) != len(CALLERS):
|
|
raise ValueError(f"expected {len(CALLERS)} caller values, found {len(values)}")
|
|
return values
|
|
|
|
|
|
def sum_rows(rows: list[dict], fields: tuple[str, ...], lod_max: int | None) -> dict:
|
|
selected = [row for row in rows if lod_max is None or row["lod"] <= lod_max]
|
|
result = {field: sum(row[field] for row in selected) for field in fields}
|
|
for field in ARRAY_FIELDS:
|
|
result[field] = [sum(row[field][i] for row in selected) for i in range(len(CALLERS))]
|
|
result["tiles"] = len(selected)
|
|
return result
|
|
|
|
|
|
def analyze_run(run: dict) -> tuple[dict, list[dict], list[dict], list[dict]]:
|
|
path = ROOT / run["log"]
|
|
content = path.read_text(encoding="utf-8", errors="replace")
|
|
sampler_match = SAMPLER.search(content)
|
|
stamp_match = STAMP.search(content)
|
|
pipeline_match = PIPELINE.search(content)
|
|
if sampler_match is None or stamp_match is None or pipeline_match is None:
|
|
raise RuntimeError(f"sampler, pipeline, or source stamp missing: {path}")
|
|
pipeline = kv(pipeline_match.group(1))
|
|
if int(pipeline.get("workers", 0)) != WORKERS:
|
|
raise RuntimeError(f"expected W{WORKERS} census run: {path}: {pipeline}")
|
|
sampler = kv(sampler_match.group(1))
|
|
if sampler.get("status") != "PASS" or int(sampler.get("lost", "-1")) != 0:
|
|
raise RuntimeError(f"sampler loss or failure: {path} {sampler}")
|
|
samples = [
|
|
{"pc": pc, "lod": int(lod), "stage": stage, "route": route,
|
|
"phase": phase, "halo": int(halo), "weight": int(weight)}
|
|
for pc, lod, stage, route, phase, halo, weight in SAMPLE.findall(content)
|
|
]
|
|
stored = int(sampler["samples"])
|
|
total_weight = sum(row["weight"] for row in samples)
|
|
expanded = int(sampler["expanded_ticks"])
|
|
if len(samples) != stored or total_weight != expanded:
|
|
raise RuntimeError(f"sample record closure failed: {path} rows={len(samples)}/{stored} "
|
|
f"weight={total_weight}/{expanded}")
|
|
scale_s = float(sampler["profiled_thread_cpu_s"]) / expanded
|
|
|
|
cpu_rows, composer_rows, worker_rows, detail_rows = [], [], [], []
|
|
for lod, tail in CPU_LINE.findall(content):
|
|
values = kv(tail)
|
|
row = {"lod": int(lod)}
|
|
row.update({name: int(values.get(name, 0)) for name in CPU_FIELDS})
|
|
row.update({name: array_value(values.get(name)) for name in ARRAY_FIELDS})
|
|
cpu_rows.append(row)
|
|
for lod, tail in COMPOSER_LINE.findall(content):
|
|
values = kv(tail)
|
|
row = {"lod": int(lod)}
|
|
row.update({name: int(values.get(name, 0)) for name in COMPOSER_FIELDS})
|
|
composer_rows.append(row)
|
|
for regex, destination, names in (
|
|
(WORKER_LINE, worker_rows, ("tiles", "worker_sum_s", "column_eval_exclusive_ms",
|
|
"mc_exclusive_ms", "ground_refine_exclusive_ms",
|
|
"cap_refine_exclusive_ms", "skirt_ms", "mesh_assembly_ms")),
|
|
(DETAIL_LINE, detail_rows, ("tiles", "mc_exclusive_ms", "mc_grid_ms",
|
|
"mc_cell_class_ms", "mc_gradient_ms", "mc_interpolation_ms",
|
|
"mc_dedup_stream_ms", "mc_other_ms", "ground_exclusive_ms",
|
|
"cap_exclusive_ms", "stratum_shell_ms", "connector_shell_ms",
|
|
"stratum_column_compute_ms", "connector_column_compute_ms")),
|
|
):
|
|
for lod, tail in regex.findall(content):
|
|
values = kv(tail)
|
|
row = {"lod": int(lod)}
|
|
for name in names:
|
|
if name == "tiles":
|
|
row[name] = int(values.get(name, 0))
|
|
else:
|
|
row[name] = float(values.get(name, 0))
|
|
destination.append(row)
|
|
if not cpu_rows or not composer_rows or not detail_rows:
|
|
raise RuntimeError(f"instrumented tile output missing: {path}")
|
|
|
|
near_samples = [row for row in samples if row["lod"] <= 2]
|
|
all_phase = Counter()
|
|
near_phase = Counter()
|
|
near_route_phase = Counter()
|
|
near_stage_phase = Counter()
|
|
near_lod_stage_phase = Counter()
|
|
for sample in samples:
|
|
all_phase[sample["phase"]] += sample["weight"]
|
|
if sample["lod"] <= 2:
|
|
near_phase[sample["phase"]] += sample["weight"]
|
|
near_route_phase[(sample["route"], sample["phase"])] += sample["weight"]
|
|
near_stage_phase[(sample["stage"], sample["phase"])] += sample["weight"]
|
|
near_lod_stage_phase[(sample["lod"], sample["stage"], sample["phase"])] += sample["weight"]
|
|
|
|
near_cpu = sum_rows(cpu_rows, CPU_FIELDS, 2)
|
|
all_cpu = sum_rows(cpu_rows, CPU_FIELDS, None)
|
|
near_composer = {field: sum(row[field] for row in composer_rows if row["lod"] <= 2)
|
|
for field in COMPOSER_FIELDS}
|
|
all_composer = {field: sum(row[field] for row in composer_rows) for field in COMPOSER_FIELDS}
|
|
near_workers = [row for row in worker_rows if row["lod"] <= 2]
|
|
near_details = [row for row in detail_rows if row["lod"] <= 2]
|
|
direct_worker_s = sum(row["worker_sum_s"] for row in near_workers)
|
|
direct_grid_s = sum(row["mc_grid_ms"] for row in near_details) / 1000.0
|
|
sample_near_cpu_s = sum(near_phase.values()) * scale_s
|
|
sample_grid_s = near_stage_phase[("grid_fill", "base_field")] * scale_s
|
|
sample_grid_allphase_s = sum(
|
|
count for (stage, _phase), count in near_stage_phase.items() if stage == "grid_fill") * scale_s
|
|
near_total_ticks = sum(near_phase.values())
|
|
near_phase_cpu_s = {phase: direct_worker_s * ticks / near_total_ticks
|
|
for phase, ticks in near_phase.items()} if near_total_ticks else {}
|
|
direct_worker_by_lod = {row["lod"]: row["worker_sum_s"] for row in near_workers}
|
|
sampled_grid_estimate_s = 0.0
|
|
for lod, worker_s in direct_worker_by_lod.items():
|
|
stage_ticks = sum(weight for (sample_lod, stage, _phase), weight
|
|
in near_lod_stage_phase.items()
|
|
if sample_lod == lod and stage == "grid_fill")
|
|
lod_ticks = sum(weight for (sample_lod, _stage, _phase), weight
|
|
in near_lod_stage_phase.items() if sample_lod == lod)
|
|
if lod_ticks:
|
|
sampled_grid_estimate_s += worker_s * stage_ticks / lod_ticks
|
|
# Compare the sample-derived per-LOD grid-fill share with direct mc_grid_ms.
|
|
grid_closure_pct = 100.0 * sampled_grid_estimate_s / direct_grid_s if direct_grid_s else 0.0
|
|
near_direct = {name: sum(row[name] for row in near_details)
|
|
for name in ("mc_exclusive_ms", "mc_grid_ms", "mc_cell_class_ms",
|
|
"mc_gradient_ms", "mc_interpolation_ms", "mc_dedup_stream_ms",
|
|
"mc_other_ms", "ground_exclusive_ms", "cap_exclusive_ms",
|
|
"stratum_shell_ms", "connector_shell_ms",
|
|
"stratum_column_compute_ms", "connector_column_compute_ms")}
|
|
all_direct = {name: sum(row[name] for row in detail_rows)
|
|
for name in ("stratum_shell_ms", "connector_shell_ms",
|
|
"stratum_column_compute_ms", "connector_column_compute_ms")}
|
|
|
|
skip_fields = ("sweep_exact_fallback_by_caller", "sweep_bound_skip_by_caller",
|
|
"sweep_tile_free_skip_by_caller", "sweep_vertical_bound_skip_by_caller",
|
|
"sweep_negative_reach_skip_by_caller", "sweep_positive_reach_skip_by_caller")
|
|
unresolved = [near_cpu["sweep_missing_by_caller"][i]
|
|
- sum(near_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))]
|
|
all_unresolved = [all_cpu["sweep_missing_by_caller"][i]
|
|
- sum(all_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))]
|
|
skirt_checks = {
|
|
caller: near_cpu["density_offgrid_float_by_caller"][CALLERS.index(caller)]
|
|
== near_cpu[probe]
|
|
for caller, probe in SKIRT_CALLERS.items()
|
|
}
|
|
validation = {
|
|
"sampler_sample_count_and_expanded_tick_sum": True,
|
|
"all_sweep_misses_match_composer": sum(all_cpu["sweep_missing_by_caller"])
|
|
== all_composer["sweep_missing"],
|
|
"all_misses_have_exact_or_skip_disposition": all(value == 0 for value in all_unresolved),
|
|
"near_misses_have_exact_or_skip_disposition": all(value == 0 for value in unresolved),
|
|
"near_midpoint_unique_matches_unclamped_offgrid":
|
|
near_cpu["midpoint_attempts"] - near_cpu["midpoint_memo_hits"]
|
|
== near_cpu["density_offgrid_unclamped_by_caller"][CALLERS.index("mc_midpoint_correction")],
|
|
"near_skirt_callers_match_offgrid_float": all(skirt_checks.values()),
|
|
"near_skirt_caller_checks": skirt_checks,
|
|
"grid_fill_closure_diagnostic_recorded": direct_grid_s > 0,
|
|
}
|
|
base_samples = near_composer["sweep_base_samples"]
|
|
detail_samples = near_composer["sweep_detail_samples"]
|
|
total_sweep_samples = base_samples + detail_samples
|
|
row = {
|
|
"stratum": run["stratum"], "replicate": run["replicate"],
|
|
"workers": WORKERS, "source_stamp": stamp_match.group(1),
|
|
"exit_code": run["exit_code"],
|
|
"sampler_samples": stored, "expanded_ticks": expanded,
|
|
"sampler_lost": int(sampler["lost"]), "timer_overruns": int(sampler["timer_overruns"]),
|
|
"timer_threads": int(sampler["timer_threads"]),
|
|
"profiled_thread_cpu_s": float(sampler["profiled_thread_cpu_s"]),
|
|
"whole_process_cpu_s": float(sampler["whole_process_cpu_s"]),
|
|
"handler_cpu_ms": float(sampler["handler_cpu_ms"]),
|
|
"handler_share_pct": float(sampler["handler_share_pct"]),
|
|
"observed_us_per_tick": float(sampler["observed_us_per_tick"]),
|
|
"near_lod_samples": len(near_samples),
|
|
"near_lod_weighted_ticks": sum(near_phase.values()),
|
|
"near_sampled_worker_cpu_s": sample_near_cpu_s,
|
|
"near_direct_worker_cpu_s": direct_worker_s,
|
|
"near_sample_to_direct_worker_pct": 100.0 * sample_near_cpu_s / direct_worker_s if direct_worker_s else 0.0,
|
|
"near_sampled_grid_fill_s": sample_grid_allphase_s,
|
|
"near_grid_fill_share_estimate_s": sampled_grid_estimate_s,
|
|
"near_phase_worker_cpu_s": near_phase_cpu_s,
|
|
"near_phase_worker_cpu_pct": {phase: 100.0 * ticks / near_total_ticks
|
|
for phase, ticks in near_phase.items()}
|
|
if near_total_ticks else {},
|
|
"near_direct_grid_fill_s": direct_grid_s,
|
|
"near_grid_fill_closure_pct": grid_closure_pct,
|
|
"near_grid_fill_base_field_samples_s": sample_grid_s,
|
|
"near_phase_ticks": dict(near_phase),
|
|
"near_route_phase_ticks": {f"{r}|{p}": count for (r, p), count in near_route_phase.items()},
|
|
"near_stage_phase_ticks": {f"{s}|{p}": count for (s, p), count in near_stage_phase.items()},
|
|
"all_phase_ticks": dict(all_phase),
|
|
"near_direct_stage_ms": near_direct,
|
|
"all_direct_far_shell_ms": all_direct,
|
|
"near_cpu_counts": near_cpu, "all_cpu_counts": all_cpu,
|
|
"near_composer_counts": near_composer, "all_composer_counts": all_composer,
|
|
"near_sweep_base_samples": base_samples, "near_sweep_detail_samples": detail_samples,
|
|
"near_sweep_base_share_pct": 100.0 * base_samples / total_sweep_samples if total_sweep_samples else 0.0,
|
|
"near_sweep_detail_share_pct": 100.0 * detail_samples / total_sweep_samples if total_sweep_samples else 0.0,
|
|
"all_sweep_base_samples": all_composer["sweep_base_samples"],
|
|
"all_sweep_detail_samples": all_composer["sweep_detail_samples"],
|
|
"validation": validation,
|
|
"run_log": run["log"],
|
|
}
|
|
phase_rows = []
|
|
near_total = sum(near_phase.values())
|
|
for phase, ticks in sorted(near_phase.items()):
|
|
phase_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
|
"phase": phase, "weighted_ticks": ticks,
|
|
"sampled_cpu_s": ticks * scale_s,
|
|
"share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0,
|
|
"direct_worker_cpu_s_estimate": near_phase_cpu_s.get(phase, 0.0),
|
|
"share_near_direct_worker_cpu_pct":
|
|
100 * ticks / near_total if near_total else 0})
|
|
route_rows = []
|
|
for (route, phase), ticks in sorted(near_route_phase.items()):
|
|
route_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
|
"route": route, "phase": phase, "weighted_ticks": ticks,
|
|
"sampled_cpu_s": ticks * scale_s,
|
|
"share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0,
|
|
"direct_worker_cpu_s_estimate":
|
|
direct_worker_s * ticks / near_total if near_total else 0.0,
|
|
"share_near_direct_worker_cpu_pct":
|
|
100 * ticks / near_total if near_total else 0})
|
|
caller_rows = []
|
|
for i, caller in enumerate(CALLERS):
|
|
caller_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
|
"caller": caller,
|
|
**{name.removesuffix("_by_caller"): near_cpu[name][i]
|
|
for name in ARRAY_FIELDS}})
|
|
return row, phase_rows, route_rows, caller_rows
|
|
|
|
|
|
def write_csv(path: Path, rows: list[dict]) -> None:
|
|
if rows:
|
|
with path.open("w", encoding="utf-8", newline="") as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def main() -> int:
|
|
runs = json.loads((OUT / "runs.json").read_text(encoding="utf-8"))
|
|
details, phase_rows, route_rows, caller_rows = [], [], [], []
|
|
for run in runs:
|
|
detail, phases, routes, callers = analyze_run(run)
|
|
details.append(detail)
|
|
phase_rows.extend(phases)
|
|
route_rows.extend(routes)
|
|
caller_rows.extend(callers)
|
|
means = []
|
|
for stratum in ("SW", "FI", "TN", "VS"):
|
|
subset = [row for row in details if row["stratum"] == stratum]
|
|
for metric in ("near_sampled_worker_cpu_s", "near_direct_worker_cpu_s",
|
|
"near_sample_to_direct_worker_pct", "near_sampled_grid_fill_s",
|
|
"near_grid_fill_share_estimate_s", "near_direct_grid_fill_s",
|
|
"near_grid_fill_closure_pct",
|
|
"near_sweep_base_share_pct", "near_sweep_detail_share_pct"):
|
|
means.append({"stratum": stratum, "metric": metric,
|
|
"runs": len(subset), "mean": statistics.mean(row[metric] for row in subset),
|
|
"median": statistics.median(row[metric] for row in subset),
|
|
"min": min(row[metric] for row in subset),
|
|
"max": max(row[metric] for row in subset)})
|
|
write_csv(OUT / "phase-cost.csv", phase_rows)
|
|
write_csv(OUT / "route-phase-cost.csv", route_rows)
|
|
write_csv(OUT / "caller-dispositions-near.csv", caller_rows)
|
|
summary = {"runs": details, "metrics": means,
|
|
"validation_pass": all(
|
|
all(value for key, value in row["validation"].items()
|
|
if key != "near_skirt_caller_checks")
|
|
and all(row["validation"]["near_skirt_caller_checks"].values())
|
|
for row in details)}
|
|
(OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
|
print("[L103CensusSummary] " + json.dumps(
|
|
{"validation_pass": summary["validation_pass"], "metrics": means}, sort_keys=True), flush=True)
|
|
return 0 if summary["validation_pass"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|