#!/usr/bin/env python3 """Close L103 sampler costs against direct grid and tile-worker timings.""" from __future__ import annotations import csv import json import re import statistics from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[4] OUT = ROOT / "Saved/Linux/l103/census" WORKERS = 12 CALLERS = ( "unknown", "density_grid", "gradient_normal_fallback", "mc_midpoint_correction", "skirt_surface_vertex", "skirt_extruded_vertex", "skirt_clipped_vertex", "skirt_refinement_face", "skirt_coarse_centroid", "far_shell_surface_query", "far_shell_projection_normal", "far_shell_stratum_refinement", ) SKIRT_CALLERS = { "skirt_surface_vertex": "skirt_surface_vertex_offgrid", "skirt_extruded_vertex": "skirt_extruded_vertex_offgrid", "skirt_clipped_vertex": "skirt_clipped_vertex_offgrid", "skirt_refinement_face": "skirt_refinement_face_offgrid", "skirt_coarse_centroid": "skirt_coarse_centroid_offgrid", } ARRAY_FIELDS = ( "sweep_missing_by_caller", "sweep_exact_fallback_by_caller", "sweep_bound_skip_by_caller", "sweep_tile_free_skip_by_caller", "sweep_vertical_bound_skip_by_caller", "sweep_negative_reach_skip_by_caller", "sweep_positive_reach_skip_by_caller", "density_offgrid_float_by_caller", "density_offgrid_unclamped_by_caller", "density_offgrid_detail_by_caller", ) CPU_FIELDS = ( "density_float_points", "density_lattice_points", "density_unclamped_points", "grad_calls", "grad_memo_hits", "grad_fallback_calls", "midpoint_attempts", "midpoint_memo_hits", "skirt_density_probes", "skirt_surface_vertex", "skirt_surface_vertex_offgrid", "skirt_extruded_vertex", "skirt_extruded_vertex_offgrid", "skirt_clipped_vertex", "skirt_clipped_vertex_offgrid", "skirt_refinement_face", "skirt_refinement_face_offgrid", "skirt_coarse_centroid", "skirt_coarse_centroid_offgrid", ) COMPOSER_FIELDS = ( "queries", "boundary_band_queries", "column_distance_calls", "carve_segment_visits", "support_segment_visits", "support_capsule_checks", "later_bore_checks", "other_capsule_checks", "mouth_support_calls", "mouth_route_segment_checks", "relief_field_calls", "sweep_segments", "sweep_pairs", "sweep_lookups", "sweep_cache_hits", "sweep_missing", "sweep_base_samples", "sweep_detail_samples", "sweep_grid_samples", "sweep_no_influence_samples", ) SAMPLE = re.compile( r"^\[L77Sample\] pc=(0x\w+) lod=(\d+) stage=(\S+) route=(\S+) " r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M) SAMPLER = re.compile(r"^\[L77Sampler\] action=stop (.*)$", re.M) FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") CPU_LINE = re.compile(r"^\[CpuTileStages\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M) COMPOSER_LINE = re.compile(r"^\[PassageComposerTile\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M) WORKER_LINE = re.compile(r"^\[WorldAssemblerLODWorker\] lod=(\d+) (.*)$", re.M) DETAIL_LINE = re.compile(r"^\[WorldAssemblerLODCostDetail\] lod=(\d+) (.*)$", re.M) PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M) def kv(payload: str) -> dict[str, str]: return dict(FIELDS.findall(payload)) def array_value(text: str | None) -> list[int]: values = [int(value) for value in text.split(",")] if text else [] if len(values) != len(CALLERS): raise ValueError(f"expected {len(CALLERS)} caller values, found {len(values)}") return values def sum_rows(rows: list[dict], fields: tuple[str, ...], lod_max: int | None) -> dict: selected = [row for row in rows if lod_max is None or row["lod"] <= lod_max] result = {field: sum(row[field] for row in selected) for field in fields} for field in ARRAY_FIELDS: result[field] = [sum(row[field][i] for row in selected) for i in range(len(CALLERS))] result["tiles"] = len(selected) return result def analyze_run(run: dict) -> tuple[dict, list[dict], list[dict], list[dict]]: path = ROOT / run["log"] content = path.read_text(encoding="utf-8", errors="replace") sampler_match = SAMPLER.search(content) stamp_match = STAMP.search(content) pipeline_match = PIPELINE.search(content) if sampler_match is None or stamp_match is None or pipeline_match is None: raise RuntimeError(f"sampler, pipeline, or source stamp missing: {path}") pipeline = kv(pipeline_match.group(1)) if int(pipeline.get("workers", 0)) != WORKERS: raise RuntimeError(f"expected W{WORKERS} census run: {path}: {pipeline}") sampler = kv(sampler_match.group(1)) if sampler.get("status") != "PASS" or int(sampler.get("lost", "-1")) != 0: raise RuntimeError(f"sampler loss or failure: {path} {sampler}") samples = [ {"pc": pc, "lod": int(lod), "stage": stage, "route": route, "phase": phase, "halo": int(halo), "weight": int(weight)} for pc, lod, stage, route, phase, halo, weight in SAMPLE.findall(content) ] stored = int(sampler["samples"]) total_weight = sum(row["weight"] for row in samples) expanded = int(sampler["expanded_ticks"]) if len(samples) != stored or total_weight != expanded: raise RuntimeError(f"sample record closure failed: {path} rows={len(samples)}/{stored} " f"weight={total_weight}/{expanded}") scale_s = float(sampler["profiled_thread_cpu_s"]) / expanded cpu_rows, composer_rows, worker_rows, detail_rows = [], [], [], [] for lod, tail in CPU_LINE.findall(content): values = kv(tail) row = {"lod": int(lod)} row.update({name: int(values.get(name, 0)) for name in CPU_FIELDS}) row.update({name: array_value(values.get(name)) for name in ARRAY_FIELDS}) cpu_rows.append(row) for lod, tail in COMPOSER_LINE.findall(content): values = kv(tail) row = {"lod": int(lod)} row.update({name: int(values.get(name, 0)) for name in COMPOSER_FIELDS}) composer_rows.append(row) for regex, destination, names in ( (WORKER_LINE, worker_rows, ("tiles", "worker_sum_s", "column_eval_exclusive_ms", "mc_exclusive_ms", "ground_refine_exclusive_ms", "cap_refine_exclusive_ms", "skirt_ms", "mesh_assembly_ms")), (DETAIL_LINE, detail_rows, ("tiles", "mc_exclusive_ms", "mc_grid_ms", "mc_cell_class_ms", "mc_gradient_ms", "mc_interpolation_ms", "mc_dedup_stream_ms", "mc_other_ms", "ground_exclusive_ms", "cap_exclusive_ms", "stratum_shell_ms", "connector_shell_ms", "stratum_column_compute_ms", "connector_column_compute_ms")), ): for lod, tail in regex.findall(content): values = kv(tail) row = {"lod": int(lod)} for name in names: if name == "tiles": row[name] = int(values.get(name, 0)) else: row[name] = float(values.get(name, 0)) destination.append(row) if not cpu_rows or not composer_rows or not detail_rows: raise RuntimeError(f"instrumented tile output missing: {path}") near_samples = [row for row in samples if row["lod"] <= 2] all_phase = Counter() near_phase = Counter() near_route_phase = Counter() near_stage_phase = Counter() near_lod_stage_phase = Counter() for sample in samples: all_phase[sample["phase"]] += sample["weight"] if sample["lod"] <= 2: near_phase[sample["phase"]] += sample["weight"] near_route_phase[(sample["route"], sample["phase"])] += sample["weight"] near_stage_phase[(sample["stage"], sample["phase"])] += sample["weight"] near_lod_stage_phase[(sample["lod"], sample["stage"], sample["phase"])] += sample["weight"] near_cpu = sum_rows(cpu_rows, CPU_FIELDS, 2) all_cpu = sum_rows(cpu_rows, CPU_FIELDS, None) near_composer = {field: sum(row[field] for row in composer_rows if row["lod"] <= 2) for field in COMPOSER_FIELDS} all_composer = {field: sum(row[field] for row in composer_rows) for field in COMPOSER_FIELDS} near_workers = [row for row in worker_rows if row["lod"] <= 2] near_details = [row for row in detail_rows if row["lod"] <= 2] direct_worker_s = sum(row["worker_sum_s"] for row in near_workers) direct_grid_s = sum(row["mc_grid_ms"] for row in near_details) / 1000.0 sample_near_cpu_s = sum(near_phase.values()) * scale_s sample_grid_s = near_stage_phase[("grid_fill", "base_field")] * scale_s sample_grid_allphase_s = sum( count for (stage, _phase), count in near_stage_phase.items() if stage == "grid_fill") * scale_s near_total_ticks = sum(near_phase.values()) near_phase_cpu_s = {phase: direct_worker_s * ticks / near_total_ticks for phase, ticks in near_phase.items()} if near_total_ticks else {} direct_worker_by_lod = {row["lod"]: row["worker_sum_s"] for row in near_workers} sampled_grid_estimate_s = 0.0 for lod, worker_s in direct_worker_by_lod.items(): stage_ticks = sum(weight for (sample_lod, stage, _phase), weight in near_lod_stage_phase.items() if sample_lod == lod and stage == "grid_fill") lod_ticks = sum(weight for (sample_lod, _stage, _phase), weight in near_lod_stage_phase.items() if sample_lod == lod) if lod_ticks: sampled_grid_estimate_s += worker_s * stage_ticks / lod_ticks # Compare the sample-derived per-LOD grid-fill share with direct mc_grid_ms. grid_closure_pct = 100.0 * sampled_grid_estimate_s / direct_grid_s if direct_grid_s else 0.0 near_direct = {name: sum(row[name] for row in near_details) for name in ("mc_exclusive_ms", "mc_grid_ms", "mc_cell_class_ms", "mc_gradient_ms", "mc_interpolation_ms", "mc_dedup_stream_ms", "mc_other_ms", "ground_exclusive_ms", "cap_exclusive_ms", "stratum_shell_ms", "connector_shell_ms", "stratum_column_compute_ms", "connector_column_compute_ms")} all_direct = {name: sum(row[name] for row in detail_rows) for name in ("stratum_shell_ms", "connector_shell_ms", "stratum_column_compute_ms", "connector_column_compute_ms")} skip_fields = ("sweep_exact_fallback_by_caller", "sweep_bound_skip_by_caller", "sweep_tile_free_skip_by_caller", "sweep_vertical_bound_skip_by_caller", "sweep_negative_reach_skip_by_caller", "sweep_positive_reach_skip_by_caller") unresolved = [near_cpu["sweep_missing_by_caller"][i] - sum(near_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))] all_unresolved = [all_cpu["sweep_missing_by_caller"][i] - sum(all_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))] skirt_checks = { caller: near_cpu["density_offgrid_float_by_caller"][CALLERS.index(caller)] == near_cpu[probe] for caller, probe in SKIRT_CALLERS.items() } validation = { "sampler_sample_count_and_expanded_tick_sum": True, "all_sweep_misses_match_composer": sum(all_cpu["sweep_missing_by_caller"]) == all_composer["sweep_missing"], "all_misses_have_exact_or_skip_disposition": all(value == 0 for value in all_unresolved), "near_misses_have_exact_or_skip_disposition": all(value == 0 for value in unresolved), "near_midpoint_unique_matches_unclamped_offgrid": near_cpu["midpoint_attempts"] - near_cpu["midpoint_memo_hits"] == near_cpu["density_offgrid_unclamped_by_caller"][CALLERS.index("mc_midpoint_correction")], "near_skirt_callers_match_offgrid_float": all(skirt_checks.values()), "near_skirt_caller_checks": skirt_checks, "grid_fill_closure_diagnostic_recorded": direct_grid_s > 0, } base_samples = near_composer["sweep_base_samples"] detail_samples = near_composer["sweep_detail_samples"] total_sweep_samples = base_samples + detail_samples row = { "stratum": run["stratum"], "replicate": run["replicate"], "workers": WORKERS, "source_stamp": stamp_match.group(1), "exit_code": run["exit_code"], "sampler_samples": stored, "expanded_ticks": expanded, "sampler_lost": int(sampler["lost"]), "timer_overruns": int(sampler["timer_overruns"]), "timer_threads": int(sampler["timer_threads"]), "profiled_thread_cpu_s": float(sampler["profiled_thread_cpu_s"]), "whole_process_cpu_s": float(sampler["whole_process_cpu_s"]), "handler_cpu_ms": float(sampler["handler_cpu_ms"]), "handler_share_pct": float(sampler["handler_share_pct"]), "observed_us_per_tick": float(sampler["observed_us_per_tick"]), "near_lod_samples": len(near_samples), "near_lod_weighted_ticks": sum(near_phase.values()), "near_sampled_worker_cpu_s": sample_near_cpu_s, "near_direct_worker_cpu_s": direct_worker_s, "near_sample_to_direct_worker_pct": 100.0 * sample_near_cpu_s / direct_worker_s if direct_worker_s else 0.0, "near_sampled_grid_fill_s": sample_grid_allphase_s, "near_grid_fill_share_estimate_s": sampled_grid_estimate_s, "near_phase_worker_cpu_s": near_phase_cpu_s, "near_phase_worker_cpu_pct": {phase: 100.0 * ticks / near_total_ticks for phase, ticks in near_phase.items()} if near_total_ticks else {}, "near_direct_grid_fill_s": direct_grid_s, "near_grid_fill_closure_pct": grid_closure_pct, "near_grid_fill_base_field_samples_s": sample_grid_s, "near_phase_ticks": dict(near_phase), "near_route_phase_ticks": {f"{r}|{p}": count for (r, p), count in near_route_phase.items()}, "near_stage_phase_ticks": {f"{s}|{p}": count for (s, p), count in near_stage_phase.items()}, "all_phase_ticks": dict(all_phase), "near_direct_stage_ms": near_direct, "all_direct_far_shell_ms": all_direct, "near_cpu_counts": near_cpu, "all_cpu_counts": all_cpu, "near_composer_counts": near_composer, "all_composer_counts": all_composer, "near_sweep_base_samples": base_samples, "near_sweep_detail_samples": detail_samples, "near_sweep_base_share_pct": 100.0 * base_samples / total_sweep_samples if total_sweep_samples else 0.0, "near_sweep_detail_share_pct": 100.0 * detail_samples / total_sweep_samples if total_sweep_samples else 0.0, "all_sweep_base_samples": all_composer["sweep_base_samples"], "all_sweep_detail_samples": all_composer["sweep_detail_samples"], "validation": validation, "run_log": run["log"], } phase_rows = [] near_total = sum(near_phase.values()) for phase, ticks in sorted(near_phase.items()): phase_rows.append({"stratum": run["stratum"], "replicate": run["replicate"], "phase": phase, "weighted_ticks": ticks, "sampled_cpu_s": ticks * scale_s, "share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0, "direct_worker_cpu_s_estimate": near_phase_cpu_s.get(phase, 0.0), "share_near_direct_worker_cpu_pct": 100 * ticks / near_total if near_total else 0}) route_rows = [] for (route, phase), ticks in sorted(near_route_phase.items()): route_rows.append({"stratum": run["stratum"], "replicate": run["replicate"], "route": route, "phase": phase, "weighted_ticks": ticks, "sampled_cpu_s": ticks * scale_s, "share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0, "direct_worker_cpu_s_estimate": direct_worker_s * ticks / near_total if near_total else 0.0, "share_near_direct_worker_cpu_pct": 100 * ticks / near_total if near_total else 0}) caller_rows = [] for i, caller in enumerate(CALLERS): caller_rows.append({"stratum": run["stratum"], "replicate": run["replicate"], "caller": caller, **{name.removesuffix("_by_caller"): near_cpu[name][i] for name in ARRAY_FIELDS}}) return row, phase_rows, route_rows, caller_rows def write_csv(path: Path, rows: list[dict]) -> None: if rows: with path.open("w", encoding="utf-8", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=list(rows[0])) writer.writeheader() writer.writerows(rows) def main() -> int: runs = json.loads((OUT / "runs.json").read_text(encoding="utf-8")) details, phase_rows, route_rows, caller_rows = [], [], [], [] for run in runs: detail, phases, routes, callers = analyze_run(run) details.append(detail) phase_rows.extend(phases) route_rows.extend(routes) caller_rows.extend(callers) means = [] for stratum in ("SW", "FI", "TN", "VS"): subset = [row for row in details if row["stratum"] == stratum] for metric in ("near_sampled_worker_cpu_s", "near_direct_worker_cpu_s", "near_sample_to_direct_worker_pct", "near_sampled_grid_fill_s", "near_grid_fill_share_estimate_s", "near_direct_grid_fill_s", "near_grid_fill_closure_pct", "near_sweep_base_share_pct", "near_sweep_detail_share_pct"): means.append({"stratum": stratum, "metric": metric, "runs": len(subset), "mean": statistics.mean(row[metric] for row in subset), "median": statistics.median(row[metric] for row in subset), "min": min(row[metric] for row in subset), "max": max(row[metric] for row in subset)}) write_csv(OUT / "phase-cost.csv", phase_rows) write_csv(OUT / "route-phase-cost.csv", route_rows) write_csv(OUT / "caller-dispositions-near.csv", caller_rows) summary = {"runs": details, "metrics": means, "validation_pass": all( all(value for key, value in row["validation"].items() if key != "near_skirt_caller_checks") and all(row["validation"]["near_skirt_caller_checks"].values()) for row in details)} (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") print("[L103CensusSummary] " + json.dumps( {"validation_pass": summary["validation_pass"], "metrics": means}, sort_keys=True), flush=True) return 0 if summary["validation_pass"] else 1 if __name__ == "__main__": raise SystemExit(main())