diag: corrected passage-off ceiling + cold worker/tile timelines (L87); passages ARE the gate
L86's switch left ApplyResolvedStratePassageCarves running (near-tile service unchanged at ~21 worker-s), so its "no change" ceiling was wrong. Corrected (bands + carves off): FI LOD0-2 2.34 -> 0.66 s, whole 3.47 -> 1.49 s @W12; TN 2.27 -> 0.74 / 3.28 -> 1.52 s. Near-tile service 15.6-22.5 -> 2.6-3.8 worker-s. Critical chain: shape/base compile -> LOD0-1 -> far-parent build (107-153 ms, pipeline thread) -> LOD2. Workers 81-85% busy in the direct window. No production scheduling change; hashes equal L85. Fast tier PASS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
1 parent
4f5f6667ae
commit
0fe7f371f7
8 files changed
+797
-20
No files matched your search
@@ -2255,8 +2255,10 @@ public:
|
|||||||
// the generic stratum transition distance creates a discontinuity at a
|
// the generic stratum transition distance creates a discontinuity at a
|
||||||
// mouth that crosses a wide transition band. Keep that opt-in field whole;
|
// mouth that crosses a wide transition band. Keep that opt-in field whole;
|
||||||
// its host-rock value already supplies the ceiling and bedrock.
|
// its host-rock value already supplies the ceiling and bedrock.
|
||||||
|
const bool disablePassageComposition=!nativeLattice
|
||||||
|
&&VF_IsL86PassageCompositionDisabled();
|
||||||
float boundary=graphDensity;
|
float boundary=graphDensity;
|
||||||
if(!nativeLattice && !VF_IsL86PassageCompositionDisabled())
|
if(!nativeLattice&&!disablePassageComposition)
|
||||||
{
|
{
|
||||||
VoxelForgeCore::Linux77Profiler::FScopedPhase l77BandsPhase(
|
VoxelForgeCore::Linux77Profiler::FScopedPhase l77BandsPhase(
|
||||||
VoxelForgeCore::Linux77Profiler::EPhase::Bands);
|
VoxelForgeCore::Linux77Profiler::EPhase::Bands);
|
||||||
@@ -2267,7 +2269,7 @@ public:
|
|||||||
// supported spine. Reapplying legacy passage carves here can erase the
|
// supported spine. Reapplying legacy passage carves here can erase the
|
||||||
// tread immediately above a broad legacy mouth or support column.
|
// tread immediately above a broad legacy mouth or support column.
|
||||||
float composed=boundary;
|
float composed=boundary;
|
||||||
if(!nativeLattice)
|
if(!nativeLattice&&!disablePassageComposition)
|
||||||
{
|
{
|
||||||
VoxelForgeCore::Linux77Profiler::FScopedPhase l77PassagePhase(
|
VoxelForgeCore::Linux77Profiler::FScopedPhase l77PassagePhase(
|
||||||
VoxelForgeCore::Linux77Profiler::EPhase::Passage);
|
VoxelForgeCore::Linux77Profiler::EPhase::Passage);
|
||||||
|
|||||||
@@ -0,0 +1,271 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Turn L87 worker traces into tile and worker interval tables."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import statistics
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
DEFAULT_DIR = ROOT / "Saved/Linux/l87/timeline/final-phase"
|
||||||
|
EVENT = re.compile(r"^\[CpuTileTimeline\] event=(\S+) (.*)$", re.M)
|
||||||
|
COLD = re.compile(r"^\[ColdTimeline\] event=(\S+) edge=(\S+) at_ms=([0-9.]+) cpu_ms=([0-9.]+)$", re.M)
|
||||||
|
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 .* cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
PIPELINE = re.compile(r"^\[WorldAssemblerNearLODPipelineReady\] lods=0-2 .* cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
WORKER_COUNT = re.compile(r"^\[HeadlessConfig\] .*\bworkers=(\d+)", re.M)
|
||||||
|
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||||
|
KEY = re.compile(r"key=\((-?\d+),(-?\d+),(-?\d+),lod(-?\d+)\)")
|
||||||
|
TILE_EVENTS = {"requested", "admitted", "started", "finished", "published", "drained"}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_event(match: re.Match[str]) -> dict:
|
||||||
|
event, payload = match.groups()
|
||||||
|
fields = dict(FIELDS.findall(payload))
|
||||||
|
key_match = KEY.search(payload)
|
||||||
|
key = None
|
||||||
|
if key_match:
|
||||||
|
x, y, z, lod = (int(value) for value in key_match.groups())
|
||||||
|
key = (x, y, z, lod)
|
||||||
|
fields["event"] = event
|
||||||
|
fields["key"] = key
|
||||||
|
fields["at_ms"] = float(fields["at_ms"])
|
||||||
|
fields["worker_id"] = int(fields["worker_id"])
|
||||||
|
return fields
|
||||||
|
|
||||||
|
|
||||||
|
def clip_interval(start: float, end: float, window_start: float,
|
||||||
|
window_end: float) -> tuple[float, float] | None:
|
||||||
|
start, end = max(start, window_start), min(end, window_end)
|
||||||
|
return (start, end) if end > start else None
|
||||||
|
|
||||||
|
|
||||||
|
def merge_duration(intervals: list[tuple[float, float]]) -> float:
|
||||||
|
if not intervals:
|
||||||
|
return 0.0
|
||||||
|
intervals = sorted(intervals)
|
||||||
|
total, lo, hi = 0.0, intervals[0][0], intervals[0][1]
|
||||||
|
for start, end in intervals[1:]:
|
||||||
|
if start <= hi:
|
||||||
|
hi = max(hi, end)
|
||||||
|
else:
|
||||||
|
total += hi - lo
|
||||||
|
lo, hi = start, end
|
||||||
|
return total + hi - lo
|
||||||
|
|
||||||
|
|
||||||
|
def analyze(log: Path) -> tuple[dict, list[dict], list[dict], list[dict]]:
|
||||||
|
content = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
events = [parse_event(match) for match in EVENT.finditer(content)]
|
||||||
|
if not events:
|
||||||
|
raise RuntimeError(f"no worker timeline events in {log}")
|
||||||
|
cold = {(event, edge): (float(at), float(cpu))
|
||||||
|
for event, edge, at, cpu in COLD.findall(content)}
|
||||||
|
requested = cold[("desired_tile_set", "requested")]
|
||||||
|
done = cold[("lod0_2_done", "pipeline_drain")]
|
||||||
|
window_start, window_end = requested[0], done[0]
|
||||||
|
window_ms = window_end - window_start
|
||||||
|
workers_match = WORKER_COUNT.search(content)
|
||||||
|
if workers_match is None:
|
||||||
|
raise RuntimeError(f"worker count missing in {log}")
|
||||||
|
worker_count = int(workers_match.group(1))
|
||||||
|
near_ready = NEAR.search(content)
|
||||||
|
pipeline_ready = PIPELINE.search(content)
|
||||||
|
if near_ready is None or pipeline_ready is None:
|
||||||
|
raise RuntimeError(f"near-ready marks missing in {log}")
|
||||||
|
tiles: dict[tuple[int, int, int, int], dict] = defaultdict(dict)
|
||||||
|
for event in events:
|
||||||
|
key = event["key"]
|
||||||
|
if key is not None and key[3] <= 2 and event["event"] in TILE_EVENTS:
|
||||||
|
tiles[key][event["event"]] = event
|
||||||
|
|
||||||
|
tile_rows = []
|
||||||
|
for key in sorted(tiles, key=lambda item: (item[3], item[2], item[1], item[0])):
|
||||||
|
row = tiles[key]
|
||||||
|
timestamps = {name: row.get(name, {}).get("at_ms") for name in
|
||||||
|
("requested", "admitted", "started", "finished", "published", "drained")}
|
||||||
|
if any(timestamps[name] is None for name in
|
||||||
|
("requested", "admitted", "started", "finished", "published")):
|
||||||
|
raise RuntimeError(f"incomplete per-tile lifecycle for {key} in {log}")
|
||||||
|
tile_rows.append({
|
||||||
|
"log": log.name, "x": key[0], "y": key[1], "z": key[2], "lod": key[3],
|
||||||
|
**timestamps,
|
||||||
|
"request_to_admit_ms": round(timestamps["admitted"] - timestamps["requested"], 3),
|
||||||
|
"admit_to_start_ms": round(timestamps["started"] - timestamps["admitted"], 3),
|
||||||
|
"start_to_finish_ms": round(timestamps["finished"] - timestamps["started"], 3),
|
||||||
|
"finish_to_publish_ms": round(timestamps["published"] - timestamps["finished"], 3),
|
||||||
|
"request_to_publish_ms": round(timestamps["published"] - timestamps["requested"], 3),
|
||||||
|
"worker_id": row["started"]["worker_id"],
|
||||||
|
})
|
||||||
|
|
||||||
|
by_worker_busy: dict[int, list[tuple[float, float]]] = defaultdict(list)
|
||||||
|
busy_rows = []
|
||||||
|
for key, row in tiles.items():
|
||||||
|
start = row.get("started")
|
||||||
|
end = row.get("published")
|
||||||
|
if start is None or end is None:
|
||||||
|
continue
|
||||||
|
clipped = clip_interval(start["at_ms"], end["at_ms"], window_start, window_end)
|
||||||
|
if clipped is None:
|
||||||
|
continue
|
||||||
|
wid = start["worker_id"]
|
||||||
|
by_worker_busy[wid].append(clipped)
|
||||||
|
busy_rows.append({
|
||||||
|
"log": log.name, "interval": "busy", "worker_id": wid,
|
||||||
|
"tile": f"({key[0]},{key[1]},{key[2]},lod{key[3]})",
|
||||||
|
"start_ms": round(clipped[0], 3), "end_ms": round(clipped[1], 3),
|
||||||
|
"duration_ms": round(clipped[1] - clipped[0], 3),
|
||||||
|
})
|
||||||
|
|
||||||
|
idle_starts: dict[int, list[float]] = defaultdict(list)
|
||||||
|
idle_intervals: dict[int, list[tuple[float, float]]] = defaultdict(list)
|
||||||
|
idle_rows = []
|
||||||
|
for event in sorted(events, key=lambda item: item["at_ms"]):
|
||||||
|
wid = event["worker_id"]
|
||||||
|
if event["event"] == "idle_begin":
|
||||||
|
idle_starts[wid].append(event["at_ms"])
|
||||||
|
elif event["event"] == "idle_end" and idle_starts[wid]:
|
||||||
|
start = idle_starts[wid].pop(0)
|
||||||
|
clipped = clip_interval(start, event["at_ms"], window_start, window_end)
|
||||||
|
if clipped is not None:
|
||||||
|
idle_intervals[wid].append(clipped)
|
||||||
|
idle_rows.append({
|
||||||
|
"log": log.name, "interval": "idle", "worker_id": wid, "tile": "",
|
||||||
|
"start_ms": round(clipped[0], 3), "end_ms": round(clipped[1], 3),
|
||||||
|
"duration_ms": round(clipped[1] - clipped[0], 3),
|
||||||
|
})
|
||||||
|
|
||||||
|
worker_summary_rows = []
|
||||||
|
busy_total = idle_total = 0.0
|
||||||
|
for wid in range(worker_count):
|
||||||
|
busy_ms = merge_duration(by_worker_busy[wid])
|
||||||
|
idle_ms = merge_duration(idle_intervals[wid])
|
||||||
|
busy_total += busy_ms
|
||||||
|
idle_total += idle_ms
|
||||||
|
worker_summary_rows.append({
|
||||||
|
"log": log.name, "worker_id": wid, "window_ms": round(window_ms, 3),
|
||||||
|
"busy_ms": round(busy_ms, 3), "idle_ms": round(idle_ms, 3),
|
||||||
|
"other_ms": round(max(0.0, window_ms - busy_ms - idle_ms), 3),
|
||||||
|
"busy_pct": round(100.0 * busy_ms / window_ms, 2),
|
||||||
|
"idle_pct": round(100.0 * idle_ms / window_ms, 2),
|
||||||
|
})
|
||||||
|
worker_interval_rows = busy_rows + idle_rows
|
||||||
|
|
||||||
|
phase_names = (
|
||||||
|
("shape_compile", "begin"), ("base_compile", "begin"),
|
||||||
|
("base_compile", "end"), ("shape_compile", "end"),
|
||||||
|
("lod0_1_admission", "begin"),
|
||||||
|
("lod0_1_admission", "end"), ("lod0_1_done", "publish_drained"),
|
||||||
|
("far_parent_build", "begin"), ("far_parent_build", "end"),
|
||||||
|
("lod2_queue", "begin"), ("lod2_queue", "end"),
|
||||||
|
("lod2_done", "pipeline_drain"), ("lod0_2_done", "pipeline_drain"),
|
||||||
|
)
|
||||||
|
phases = {}
|
||||||
|
for name, edge in phase_names:
|
||||||
|
if (name, edge) in cold:
|
||||||
|
at, cpu = cold[(name, edge)]
|
||||||
|
phases[f"{name}_{edge}"] = {"wall_ms": round(at, 3), "cpu_ms": round(cpu, 3)}
|
||||||
|
cpu_start = requested[1]
|
||||||
|
cpu_end = done[1]
|
||||||
|
lod01_end = cold[("lod0_1_done", "publish_drained")]
|
||||||
|
lod2_end = cold[("lod2_done", "pipeline_drain")]
|
||||||
|
last = {}
|
||||||
|
for name in ("finished", "published", "drained"):
|
||||||
|
latest = max((row.get(name) for row in tiles.values() if row.get(name)),
|
||||||
|
key=lambda item: item["at_ms"], default=None)
|
||||||
|
last[name] = ({"at_ms": round(latest["at_ms"], 3),
|
||||||
|
"tile": f"{latest['key'][0]},{latest['key'][1]},{latest['key'][2]},lod{latest['key'][3]}"}
|
||||||
|
if latest else None)
|
||||||
|
lod01_done_ms = cold[("lod0_1_done", "publish_drained")][0]
|
||||||
|
lod2_queue_begin_ms = cold[("lod2_queue", "begin")][0]
|
||||||
|
lod2_done_ms = lod2_end[0]
|
||||||
|
far_begin_ms = cold[("far_parent_build", "begin")][0]
|
||||||
|
far_end_ms = cold[("far_parent_build", "end")][0]
|
||||||
|
tile_latency = {}
|
||||||
|
for lod in range(3):
|
||||||
|
rows = [row for row in tile_rows if row["lod"] == lod]
|
||||||
|
tile_latency[str(lod)] = {"tiles": len(rows)}
|
||||||
|
for metric in ("request_to_admit_ms", "admit_to_start_ms", "start_to_finish_ms",
|
||||||
|
"finish_to_publish_ms", "request_to_publish_ms"):
|
||||||
|
values = sorted(row[metric] for row in rows)
|
||||||
|
if values:
|
||||||
|
tile_latency[str(lod)][metric] = {
|
||||||
|
"median": round(statistics.median(values), 3),
|
||||||
|
"p90": round(values[min(len(values) - 1, int(0.90 * len(values)))], 3),
|
||||||
|
"p99": round(values[min(len(values) - 1, int(0.99 * len(values)))], 3),
|
||||||
|
}
|
||||||
|
summary = {
|
||||||
|
"log": str(log.resolve().relative_to(ROOT)), "worker_count": worker_count,
|
||||||
|
"requested_near_tiles": len(tile_rows),
|
||||||
|
"lod0_tiles": sum(1 for key in tiles if key[3] == 0),
|
||||||
|
"lod1_tiles": sum(1 for key in tiles if key[3] == 1),
|
||||||
|
"lod2_tiles": sum(1 for key in tiles if key[3] == 2),
|
||||||
|
"window_start_ms": round(window_start, 3), "window_end_ms": round(window_end, 3),
|
||||||
|
"window_ms": round(window_ms, 3),
|
||||||
|
"worker_pool_busy_pct": round(100.0 * busy_total / (worker_count * window_ms), 2),
|
||||||
|
"worker_pool_explicit_idle_pct": round(100.0 * idle_total / (worker_count * window_ms), 2),
|
||||||
|
"mean_busy_workers": round(busy_total / window_ms, 2),
|
||||||
|
"mean_idle_workers": round(idle_total / window_ms, 2),
|
||||||
|
"unaccounted_worker_pct": round(100.0 * max(
|
||||||
|
0.0, worker_count * window_ms - busy_total - idle_total)
|
||||||
|
/ (worker_count * window_ms), 2),
|
||||||
|
"process_cpu_ms": round(cpu_end - cpu_start, 3),
|
||||||
|
"process_cpu_core_equivalent": round((cpu_end - cpu_start) / window_ms, 2),
|
||||||
|
"lod02_ready_s": float(near_ready.group(1)),
|
||||||
|
"lod02_pipeline_ready_s": float(pipeline_ready.group(1)),
|
||||||
|
"finish_to_pipeline_drain_ms": round(
|
||||||
|
(float(pipeline_ready.group(1)) - float(near_ready.group(1))) * 1000.0, 3),
|
||||||
|
"lod01_done_ms": round(lod01_done_ms, 3),
|
||||||
|
"lod2_queue_begin_ms": round(lod2_queue_begin_ms, 3),
|
||||||
|
"lod2_done_ms": round(lod2_done_ms, 3),
|
||||||
|
"far_parent_build_ms": round(far_end_ms - far_begin_ms, 3),
|
||||||
|
"far_parent_start_minus_lod02_done_ms": round(far_begin_ms - lod2_done_ms, 3),
|
||||||
|
"lod2_queue_start_minus_far_parent_end_ms": round(lod2_queue_begin_ms - far_end_ms, 3),
|
||||||
|
"lod2_queue_after_lod01_ms": round(lod2_queue_begin_ms - lod01_done_ms, 3),
|
||||||
|
"lod2_queue_to_done_ms": round(lod2_done_ms - cold[("lod2_queue", "end")][0], 3),
|
||||||
|
"tile_latency_by_lod": tile_latency,
|
||||||
|
"last_tile_events": last, "phases": phases,
|
||||||
|
}
|
||||||
|
return summary, tile_rows, worker_interval_rows, worker_summary_rows
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--directory", type=Path, default=DEFAULT_DIR)
|
||||||
|
args = parser.parse_args()
|
||||||
|
logs = sorted(args.directory.glob("*-measured-r1.log"))
|
||||||
|
summaries, all_tiles, all_worker_rows, all_worker_summary_rows = [], [], [], []
|
||||||
|
for log in logs:
|
||||||
|
summary, tile_rows, worker_rows, worker_summary_rows = analyze(log)
|
||||||
|
summaries.append(summary)
|
||||||
|
all_tiles.extend(tile_rows)
|
||||||
|
all_worker_rows.extend(worker_rows)
|
||||||
|
all_worker_summary_rows.extend(worker_summary_rows)
|
||||||
|
print("[L87TimelineAnalysis] " + json.dumps(summary, sort_keys=True), flush=True)
|
||||||
|
if not summaries:
|
||||||
|
raise SystemExit(f"no timeline traces found in {args.directory}")
|
||||||
|
with (args.directory / "tile-lifecycle.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||||
|
fields = list(all_tiles[0])
|
||||||
|
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(all_tiles)
|
||||||
|
with (args.directory / "worker-intervals.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||||
|
fields = list(all_worker_rows[0])
|
||||||
|
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(all_worker_rows)
|
||||||
|
with (args.directory / "worker-utilization.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||||
|
fields = list(all_worker_summary_rows[0])
|
||||||
|
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(all_worker_summary_rows)
|
||||||
|
(args.directory / "timeline-analysis.json").write_text(
|
||||||
|
json.dumps({"summaries": summaries}, indent=2) + "\n", encoding="utf-8")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# Linux 87 — LOD0–2 readiness
|
||||||
|
|
||||||
|
## Finding
|
||||||
|
|
||||||
|
The L86 passage ceiling was not actually passage-off. Its flag stopped passage feature indexing and boundary bands, but `ApplyResolvedStratePassageCarves` still ran. L86 W12 near-tile worker service stayed at about 21.0 worker-s for FI and 20.5 worker-s for TN, consistent with its unchanged readiness.
|
||||||
|
|
||||||
|
With the diagnostic corrected to skip both boundary bands and passage carves, LOD0–2 readiness fell from 2.27–2.43 s to 0.66–0.74 s. Near-tile worker service fell from 15.6–22.5 worker-s to 2.6–3.8 worker-s across these runs. The corrected ceiling also removes boundary-band work, so its service reduction is broader than the L77 census's 58% passage attribution and is not a like-for-like percentage.
|
||||||
|
|
||||||
|
The secondary scheduling gate is real: far-parent construction runs synchronously on the pipeline thread after LOD0–1 drains. It took 107–153 ms in these traces. LOD2 was queued within 0.01 ms of that build ending. The near generated-ready to pipeline-drained mark gap was 0.2–5.2 ms; the standalone headless readiness path has no collision wait. Shape/base compile begins about 60–80 ms from cold start, just after the desired-set request, and ends at the compile times in the table.
|
||||||
|
|
||||||
|
## Worker timeline
|
||||||
|
|
||||||
|
Times are milliseconds from cold start. One traced run is shown for each camera, worker count and mode. “Busy/idle” is the worker-pool share of the requested-to-LOD0–2-drained window; idle means explicit worker wait intervals.
|
||||||
|
|
||||||
|
| Window | Mode | Base compile end | LOD0–1 drained | Far-parent build | LOD2 queued | LOD2 drained | Busy / idle |
|
||||||
|
|---|---|---:|---:|---:|---:|---:|---:|
|
||||||
|
| FI W8 | direct | 286 | 2,093 | 114 | 2,207 | 2,365 | 84.6% / 15.2% |
|
||||||
|
| FI W8 | passage off | 234 | 533 | 112 | 644 | 675 | 52.3% / 47.1% |
|
||||||
|
| FI W12 | direct | 340 | 2,085 | 109 | 2,194 | 2,344 | 82.8% / 17.0% |
|
||||||
|
| FI W12 | passage off | 244 | 528 | 136 | 664 | 699 | 47.9% / 50.8% |
|
||||||
|
| TN W8 | direct | 332 | 1,993 | 112 | 2,104 | 2,426 | 83.2% / 16.6% |
|
||||||
|
| TN W8 | passage off | 278 | 569 | 111 | 680 | 728 | 49.4% / 49.9% |
|
||||||
|
| TN W12 | direct | 341 | 1,788 | 107 | 1,895 | 2,186 | 80.8% / 18.8% |
|
||||||
|
| TN W12 | passage off | 273 | 547 | 153 | 701 | 763 | 44.9% / 53.8% |
|
||||||
|
|
||||||
|
The critical chain is shape/base compile → LOD0–1 tile work → far-parent build → LOD2 queue and tile work → final tile publication/drain. The LOD0–1 queue is admitted after shape compilation; admission calls take about 1–2 ms. In passage-off traces, the freed worker slots are mostly idle: non-near work accounts for at most 1.4% of the window, so far LODs do not absorb the capacity before near readiness.
|
||||||
|
|
||||||
|
Other hypotheses: shape compilation took about 0.23–0.33 s in direct traces and 0.18–0.22 s passage-off. No separate admission-cap stall appears in the event sequence. The readiness marks differ by only a few milliseconds after generation. These cold runs use a stationary camera, so L80 movement hysteresis and arrival-burst behavior were not exercised.
|
||||||
|
|
||||||
|
The CSVs contain each traced near tile's requested, admitted, started, finished, published and drained times, plus each worker's busy and idle intervals: [tile-lifecycle.csv](../../../../Saved/Linux/l87/timeline/final-hashsafe/tile-lifecycle.csv), [worker-intervals.csv](../../../../Saved/Linux/l87/timeline/final-hashsafe/worker-intervals.csv), [worker-utilization.csv](../../../../Saved/Linux/l87/timeline/final-hashsafe/worker-utilization.csv). The lifecycle file has 10,240 tile rows across the eight traces.
|
||||||
|
|
||||||
|
## Readiness
|
||||||
|
|
||||||
|
Medians of three measured runs after one warmup per window, mode and worker count. Each run completed the expected near and whole tile counts.
|
||||||
|
|
||||||
|
| Stratum | Workers | Mode | LOD0–2 ready (s) | Whole view ready (s) |
|
||||||
|
|---|---:|---|---:|---:|
|
||||||
|
| FI | 8 | direct | 2.379 | 3.604 |
|
||||||
|
| FI | 8 | passage off | 0.677 | 1.631 |
|
||||||
|
| FI | 12 | direct | 2.343 | 3.465 |
|
||||||
|
| FI | 12 | passage off | 0.663 | 1.486 |
|
||||||
|
| TN | 8 | direct | 2.426 | 3.482 |
|
||||||
|
| TN | 8 | passage off | 0.727 | 1.585 |
|
||||||
|
| TN | 12 | direct | 2.271 | 3.277 |
|
||||||
|
| TN | 12 | passage off | 0.741 | 1.524 |
|
||||||
|
|
||||||
|
## Fix, hashes and checks
|
||||||
|
|
||||||
|
The retained fix corrects the diagnostic ceiling and adds opt-in cold worker/tile timelines. No production scheduling change was retained. A trial that built far parents after LOD2 changed the canonical SurfaceWorld style 1 LOD2 hash, so the hash-safe order remains far parents before LOD2. The fast tier caught this and passed after the trial was reverted.
|
||||||
|
|
||||||
|
Direct FI and TN views match L85 per-tile and whole-view hashes: zero mismatches across 2,249 FI tiles and 2,233 TN tiles.
|
||||||
|
|
||||||
|
| Stratum | Density | Mesh | Payload | Topology |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| FI | `99ca213b21ce86e4` | `2977525405b79fff` | `3d7d2efb581710bd` | `af1916c949224871` |
|
||||||
|
| TN | `8d23c26f5c092be1` | `15c3aad2e29d9ea0` | `3d48f68d511c21df` | `5b1584676884edd8` |
|
||||||
|
|
||||||
|
Fast tier: **PASS**, 274 checks, 295.9 s. Source stamp: `ac3d8ea64e9ac4bd5526c61990070fb504528ff56dae52d33c1937327374032b`. No `VoxelWorld.cpp` change was made, so no Unreal build was needed.
|
||||||
|
|
||||||
|
## Failed runs
|
||||||
|
|
||||||
|
- The first L86-off implementation was incomplete and is excluded from the A/B table; its carver still ran.
|
||||||
|
- The asynchronous far-shape snapshot regressed FI W8 passage-off readiness to 1.159 s from 0.676 s. The parallel-bin trial raised far-shape time to 0.686 s from about 0.11 s. Both were removed.
|
||||||
|
- The LOD2-before-far trial failed the SurfaceWorld style 1 LOD2 hash check and was reverted. The final hash-safe build passes fast tier.
|
||||||
|
- The first fast-tier invocation had no verifier-named binary at its expected path and stopped before checks. The rerun used the current source-stamped L87 binary and passed.
|
||||||
|
|
||||||
|
No run exceeded 30 minutes. No commit, push or stash was made. Logs and analysis artifacts are under `Saved/Linux/l87/`.
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Verify L87 per-tile and whole-view hashes against the L85 baseline."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l87/hashes"
|
||||||
|
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||||
|
BINARY = ROOT / "Saved/Linux/l87/bin/VoxelForgeCoreHeadless-clang19"
|
||||||
|
SOURCE_STAMP = "ac3d8ea64e9ac4bd5526c61990070fb504528ff56dae52d33c1937327374032b"
|
||||||
|
BASELINE = ROOT / "Saved/Linux/l85/hash-check-final"
|
||||||
|
STRATA = {
|
||||||
|
"FI": {"id": 1, "camera": (25.8369140625, 3.19921875, -737.0), "tiles": 2249},
|
||||||
|
"TN": {"id": 2, "camera": (19.166015625, -14.2001953125, -753.3740234375), "tiles": 2233},
|
||||||
|
}
|
||||||
|
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||||
|
TILE_HASH = re.compile(
|
||||||
|
r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
|
||||||
|
r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) "
|
||||||
|
r"topology=(0x[0-9a-f]+)$", re.M)
|
||||||
|
WHOLE_HASH = re.compile(
|
||||||
|
r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) "
|
||||||
|
r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M)
|
||||||
|
|
||||||
|
|
||||||
|
def hashes_from(text: str) -> dict[tuple[int, int, int, int], tuple[str, ...]]:
|
||||||
|
return {
|
||||||
|
tuple(int(match.group(index)) for index in range(1, 5)): match.group(5, 6, 7, 8)
|
||||||
|
for match in TILE_HASH.finditer(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str) -> dict:
|
||||||
|
OUT.mkdir(parents=True, exist_ok=True)
|
||||||
|
spec = STRATA[stratum]
|
||||||
|
camera = ",".join(f"{value:.12f}" for value in spec["camera"]) + ",0,0"
|
||||||
|
log = OUT / f"{stratum.lower()}-w8.log"
|
||||||
|
command = [
|
||||||
|
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "env",
|
||||||
|
"VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", "timeout", "1740s",
|
||||||
|
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||||
|
"--stream-pipeline", "--workers=8", "--clip-radius=3", "--max-clip-level=8",
|
||||||
|
"--max-level=8", f"--camera={camera}", "--stream-scheduler-policy=3",
|
||||||
|
"--stream-hash-tiles",
|
||||||
|
]
|
||||||
|
started = time.monotonic()
|
||||||
|
with log.open("w", encoding="utf-8") as stream:
|
||||||
|
result = subprocess.run(command, cwd=ROOT, stdout=stream,
|
||||||
|
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||||
|
output = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
stamp, whole = STAMP.search(output), WHOLE_HASH.search(output)
|
||||||
|
if result.returncode or stamp is None or whole is None:
|
||||||
|
raise RuntimeError(f"hash run failed: exit={result.returncode} {log}")
|
||||||
|
if stamp.group(1) != SOURCE_STAMP:
|
||||||
|
raise RuntimeError(f"source stamp mismatch in {log}: {stamp.group(1)}")
|
||||||
|
actual_tiles = hashes_from(output)
|
||||||
|
baseline_log = BASELINE / f"l85-s{spec['id']}.log"
|
||||||
|
baseline_tiles = hashes_from(baseline_log.read_text(encoding="utf-8", errors="replace"))
|
||||||
|
mismatches = sorted(
|
||||||
|
key for key in set(actual_tiles) | set(baseline_tiles)
|
||||||
|
if actual_tiles.get(key) != baseline_tiles.get(key)
|
||||||
|
)
|
||||||
|
actual_whole = whole.group(1, 2, 3, 4, 5)
|
||||||
|
baseline_whole_match = WHOLE_HASH.search(
|
||||||
|
baseline_log.read_text(encoding="utf-8", errors="replace"))
|
||||||
|
baseline_whole = baseline_whole_match.group(1, 2, 3, 4, 5)
|
||||||
|
row = {
|
||||||
|
"stratum": stratum, "workers": 8, "exit_code": result.returncode,
|
||||||
|
"wall_s": round(time.monotonic() - started, 3), "tile_count": len(actual_tiles),
|
||||||
|
"expected_tiles": spec["tiles"], "whole_hash": actual_whole,
|
||||||
|
"baseline_whole_hash": baseline_whole, "tile_hashes_identical": not mismatches,
|
||||||
|
"whole_hash_identical": actual_whole == baseline_whole,
|
||||||
|
"mismatch_count": len(mismatches), "first_mismatches": mismatches[:12],
|
||||||
|
"source_stamp": stamp.group(1), "log": str(log.relative_to(ROOT)),
|
||||||
|
"baseline_log": str(baseline_log.relative_to(ROOT)),
|
||||||
|
}
|
||||||
|
if len(actual_tiles) != spec["tiles"] or len(baseline_tiles) != spec["tiles"]:
|
||||||
|
raise RuntimeError(f"incomplete hash view: {json.dumps(row)}")
|
||||||
|
print("[L87HashComparison] " + json.dumps(row, sort_keys=True), flush=True)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
rows = [run(stratum) for stratum in STRATA]
|
||||||
|
(OUT / "summary.json").write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8")
|
||||||
|
return 0 if all(row["tile_hashes_identical"] and row["whole_hash_identical"]
|
||||||
|
for row in rows) else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Interleaved L87 passage A/B readiness and worker timeline matrix."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import argparse
|
||||||
|
import re
|
||||||
|
import statistics
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l87/timeline/switch-fixed"
|
||||||
|
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||||
|
BINARY = ROOT / "Saved/Linux/l87/bin/VoxelForgeCoreHeadless-clang19"
|
||||||
|
SOURCE_STAMP = "ac3d8ea64e9ac4bd5526c61990070fb504528ff56dae52d33c1937327374032b"
|
||||||
|
STRATA = {
|
||||||
|
"FI": {"id": 1, "camera": (25.8369140625, 3.19921875, -737.0),
|
||||||
|
"near": 1216, "whole": 2249},
|
||||||
|
"TN": {"id": 2, "camera": (19.166015625, -14.2001953125, -753.3740234375),
|
||||||
|
"near": 1344, "whole": 2233},
|
||||||
|
}
|
||||||
|
WORKERS = (8, 12)
|
||||||
|
MODES = ("direct", "passage_off")
|
||||||
|
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||||
|
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||||
|
NEAR = re.compile(
|
||||||
|
r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$",
|
||||||
|
re.M,
|
||||||
|
)
|
||||||
|
PUBLISHED = re.compile(
|
||||||
|
r"^\[WorldAssemblerNearLODPublished\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$",
|
||||||
|
re.M,
|
||||||
|
)
|
||||||
|
PIPELINE_READY = re.compile(
|
||||||
|
r"^\[WorldAssemblerNearLODPipelineReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$",
|
||||||
|
re.M,
|
||||||
|
)
|
||||||
|
SHAPE = re.compile(
|
||||||
|
r"^\[FarShapeStage\] parent_shapes=([0-9.]+) ms spatial_bins=([0-9.]+) ms total=([0-9.]+) ms",
|
||||||
|
re.M,
|
||||||
|
)
|
||||||
|
CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M)
|
||||||
|
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||||
|
VOLUME = re.compile(r"^\[PipelineVolume\] tiles=(\d+) lod=", re.M)
|
||||||
|
TIMELINE_EVENTS = re.compile(r"^\[CpuTileTimeline\] event=(\S+)", re.M)
|
||||||
|
|
||||||
|
|
||||||
|
def command(stratum: str, workers: int, mode: str, timeline: bool) -> list[str]:
|
||||||
|
camera = ",".join(f"{v:.12f}" for v in STRATA[stratum]["camera"]) + ",0,0"
|
||||||
|
child = [
|
||||||
|
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||||
|
"--stream-pipeline", f"--workers={workers}", "--clip-radius=3",
|
||||||
|
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
|
||||||
|
"--profile-lod-work", "--stream-scheduler-policy=3",
|
||||||
|
]
|
||||||
|
if timeline:
|
||||||
|
child.append("--cold-worker-timeline")
|
||||||
|
env = ["VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0"]
|
||||||
|
if mode == "passage_off":
|
||||||
|
env.append("VOXELFORGE_L86_DISABLE_PASSAGE_COMPOSITION=1")
|
||||||
|
return [
|
||||||
|
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "env",
|
||||||
|
*env, "timeout", "1740s", *child,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str, workers: int, mode: str, phase: str,
|
||||||
|
replicate: int, timeline: bool) -> dict:
|
||||||
|
OUT.mkdir(parents=True, exist_ok=True)
|
||||||
|
rep = f"-r{replicate}" if phase == "measured" else ""
|
||||||
|
label = f"{stratum.lower()}-w{workers}-{mode}-{phase}{rep}"
|
||||||
|
log = OUT / f"{label}.log"
|
||||||
|
started = time.monotonic()
|
||||||
|
with log.open("w", encoding="utf-8") as output:
|
||||||
|
result = subprocess.run(command(stratum, workers, mode, timeline), cwd=ROOT,
|
||||||
|
stdout=output, stderr=subprocess.STDOUT,
|
||||||
|
check=False, timeout=1740)
|
||||||
|
content = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
near, published, pipeline_ready = (
|
||||||
|
NEAR.search(content), PUBLISHED.search(content), PIPELINE_READY.search(content)
|
||||||
|
)
|
||||||
|
pipeline_match, far = PIPELINE.search(content), SHAPE.search(content)
|
||||||
|
stamp_match = STAMP.search(content)
|
||||||
|
config_match, volume_match = CONFIG.search(content), VOLUME.search(content)
|
||||||
|
if result.returncode or not all((near, published, pipeline_ready, pipeline_match, far,
|
||||||
|
config_match, volume_match, stamp_match)):
|
||||||
|
raise RuntimeError(f"run failed or required timeline is missing: {log}; exit={result.returncode}")
|
||||||
|
if stamp_match.group(1) != SOURCE_STAMP:
|
||||||
|
raise RuntimeError(f"source stamp mismatch in {log}: {stamp_match.group(1)}")
|
||||||
|
expected = STRATA[stratum]
|
||||||
|
config_fields = dict(FIELDS.findall(config_match.group(1)))
|
||||||
|
if int(config_fields["workers"]) != workers:
|
||||||
|
raise RuntimeError(f"worker mismatch in {log}: {config_fields['workers']}")
|
||||||
|
if int(volume_match.group(1)) != expected["whole"]:
|
||||||
|
raise RuntimeError(f"wrong camera/strate window in {log}: {volume_match.group(1)} tiles")
|
||||||
|
if ("[L86PassageCeiling]" in content) != (mode == "passage_off"):
|
||||||
|
raise RuntimeError(f"passage switch state mismatch in {log}")
|
||||||
|
for match in (near, published, pipeline_ready):
|
||||||
|
if (int(match.group(1)), int(match.group(2))) != (expected["near"], expected["near"]):
|
||||||
|
raise RuntimeError(f"incomplete near set in {log}: {match.group(1)}/{match.group(2)}")
|
||||||
|
fields = dict(FIELDS.findall(pipeline_match.group(1)))
|
||||||
|
whole_done, whole_total = (int(v) for v in fields["whole_tiles"].split("/"))
|
||||||
|
if (whole_done, whole_total) != (expected["whole"], expected["whole"]):
|
||||||
|
raise RuntimeError(f"incomplete whole view in {log}: {whole_done}/{whole_total}")
|
||||||
|
cpu_events = TIMELINE_EVENTS.findall(content)
|
||||||
|
if timeline and not cpu_events:
|
||||||
|
raise RuntimeError(f"worker timeline missing in {log}")
|
||||||
|
row = {
|
||||||
|
"stratum": stratum, "stratum_id": expected["id"], "workers": workers,
|
||||||
|
"mode": mode, "phase": phase, "replicate": replicate,
|
||||||
|
"timeline": timeline, "headless_recipe_label": config_fields.get("recipe"),
|
||||||
|
"cpu_tile_timeline_events": len(cpu_events),
|
||||||
|
"exit_code": result.returncode, "wall_s": round(time.monotonic() - started, 3),
|
||||||
|
"source_stamp": stamp_match.group(1),
|
||||||
|
"lod02_ready_s": float(near.group(3)),
|
||||||
|
"lod02_published_s": float(published.group(3)),
|
||||||
|
"lod02_pipeline_ready_s": float(pipeline_ready.group(3)),
|
||||||
|
"whole_view_ready_s": float(fields["whole_view_ready"].removesuffix("s")),
|
||||||
|
"shape_s": float(fields["shape"].removesuffix("s")),
|
||||||
|
"far_shape_s": float(far.group(3)) / 1000.0,
|
||||||
|
"far_parent_ms": float(far.group(1)), "far_bins_ms": float(far.group(2)),
|
||||||
|
"near_tiles": expected["near"], "whole_tiles": expected["whole"],
|
||||||
|
"log": str(log.relative_to(ROOT)),
|
||||||
|
}
|
||||||
|
print("[L87TimelineRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
global OUT
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--out", type=Path, default=Path("Saved/Linux/l87/timeline/final-phase"))
|
||||||
|
args = parser.parse_args()
|
||||||
|
OUT = args.out if args.out.is_absolute() else ROOT / args.out
|
||||||
|
if not BINARY.is_file() or not DATA.is_file():
|
||||||
|
raise SystemExit(f"missing input: {BINARY} or {DATA}")
|
||||||
|
# L80 warmup: once per camera window and worker count. Rotate both mode and
|
||||||
|
# worker order across three paired measured replicates.
|
||||||
|
for stratum in STRATA:
|
||||||
|
for workers in WORKERS:
|
||||||
|
for mode in MODES:
|
||||||
|
run(stratum, workers, mode, "warmup", 0, False)
|
||||||
|
|
||||||
|
measured = []
|
||||||
|
orders = (
|
||||||
|
((8, "direct"), (8, "passage_off"), (12, "passage_off"), (12, "direct")),
|
||||||
|
((12, "direct"), (12, "passage_off"), (8, "passage_off"), (8, "direct")),
|
||||||
|
((8, "passage_off"), (8, "direct"), (12, "direct"), (12, "passage_off")),
|
||||||
|
)
|
||||||
|
for stratum in STRATA:
|
||||||
|
timeline_pending = {(workers, mode) for workers in WORKERS for mode in MODES}
|
||||||
|
for replicate, order in enumerate(orders, 1):
|
||||||
|
for workers, mode in order:
|
||||||
|
timeline = (workers, mode) in timeline_pending
|
||||||
|
if timeline:
|
||||||
|
timeline_pending.remove((workers, mode))
|
||||||
|
measured.append(run(stratum, workers, mode, "measured", replicate, timeline))
|
||||||
|
(OUT / "runs-partial.json").write_text(
|
||||||
|
json.dumps(measured, indent=2) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
summary = []
|
||||||
|
for stratum in STRATA:
|
||||||
|
for workers in WORKERS:
|
||||||
|
for mode in MODES:
|
||||||
|
rows = [row for row in measured if row["stratum"] == stratum
|
||||||
|
and row["workers"] == workers and row["mode"] == mode]
|
||||||
|
if len(rows) != 3:
|
||||||
|
raise RuntimeError(f"expected 3 measured replicates for {stratum} W{workers} {mode}")
|
||||||
|
item = {"stratum": stratum, "workers": workers, "mode": mode,
|
||||||
|
"measured_runs": 3}
|
||||||
|
for metric in ("lod02_ready_s", "lod02_published_s",
|
||||||
|
"lod02_pipeline_ready_s", "whole_view_ready_s",
|
||||||
|
"shape_s", "far_shape_s"):
|
||||||
|
values = [row[metric] for row in rows]
|
||||||
|
item[metric] = values
|
||||||
|
item[f"{metric}_median"] = statistics.median(values)
|
||||||
|
summary.append(item)
|
||||||
|
(OUT / "summary.json").write_text(
|
||||||
|
json.dumps({"runs": measured, "summary": summary}, indent=2) + "\n",
|
||||||
|
encoding="utf-8")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -49,6 +49,7 @@ namespace
|
|||||||
bool ProfileLodWork=false;
|
bool ProfileLodWork=false;
|
||||||
bool ProfilePassageComposer=false;
|
bool ProfilePassageComposer=false;
|
||||||
bool InstrumentTileDrain=false;
|
bool InstrumentTileDrain=false;
|
||||||
|
bool ColdWorkerTimeline=false;
|
||||||
bool Linux77Profile=false;
|
bool Linux77Profile=false;
|
||||||
bool ExactMidpointDensityMemo=false;
|
bool ExactMidpointDensityMemo=false;
|
||||||
bool ActiveLaneBatching=false;
|
bool ActiveLaneBatching=false;
|
||||||
@@ -1450,6 +1451,7 @@ namespace
|
|||||||
" --audit-uniform-block-proofs (fill and check every interval-certified block)\n"
|
" --audit-uniform-block-proofs (fill and check every interval-certified block)\n"
|
||||||
" --profile-lod-work (report per-LOD density sample counts)\n"
|
" --profile-lod-work (report per-LOD density sample counts)\n"
|
||||||
" --instrument-tile-drain (opt-in CpuTileTimeline and per-tile stages/counters)\n"
|
" --instrument-tile-drain (opt-in CpuTileTimeline and per-tile stages/counters)\n"
|
||||||
|
" --cold-worker-timeline (record requested/admitted/published tiles and worker idle spans)\n"
|
||||||
" --linux77-profile (Linux-only SIGPROF stack and route census)\n"
|
" --linux77-profile (Linux-only SIGPROF stack and route census)\n"
|
||||||
" --profile-passage-composer (emit per-tile passage work counters)\n"
|
" --profile-passage-composer (emit per-tile passage work counters)\n"
|
||||||
" --exact-midpoint-density-memo (exact-input midpoint query memo A/B)\n"
|
" --exact-midpoint-density-memo (exact-input midpoint query memo A/B)\n"
|
||||||
@@ -1509,6 +1511,7 @@ namespace
|
|||||||
else if(arg=="--audit-uniform-block-proofs")options.AuditUniformBlockProofs=true;
|
else if(arg=="--audit-uniform-block-proofs")options.AuditUniformBlockProofs=true;
|
||||||
else if(arg=="--profile-lod-work")options.ProfileLodWork=true;
|
else if(arg=="--profile-lod-work")options.ProfileLodWork=true;
|
||||||
else if(arg=="--instrument-tile-drain")options.InstrumentTileDrain=true;
|
else if(arg=="--instrument-tile-drain")options.InstrumentTileDrain=true;
|
||||||
|
else if(arg=="--cold-worker-timeline")options.ColdWorkerTimeline=true;
|
||||||
else if(arg=="--linux77-profile")
|
else if(arg=="--linux77-profile")
|
||||||
{options.Linux77Profile=true;options.ProfileLodWork=true;}
|
{options.Linux77Profile=true;options.ProfileLodWork=true;}
|
||||||
else if(arg=="--profile-passage-composer")
|
else if(arg=="--profile-passage-composer")
|
||||||
@@ -2273,5 +2276,5 @@ int main(int argc,char** argv)
|
|||||||
options.StreamHashAllowUniformSkips,options.Linux77Profile,
|
options.StreamHashAllowUniformSkips,options.Linux77Profile,
|
||||||
options.StreamingSchedulerPolicy,options.MovingCameraTrace,
|
options.StreamingSchedulerPolicy,options.MovingCameraTrace,
|
||||||
options.MovingCameraSpeedMetersPerSecond,options.MovingCameraDurationSeconds,
|
options.MovingCameraSpeedMetersPerSecond,options.MovingCameraDurationSeconds,
|
||||||
options.MovingCameraDwellSeconds);
|
options.MovingCameraDwellSeconds,options.ColdWorkerTimeline);
|
||||||
}
|
}
|
||||||
@@ -11,6 +11,7 @@
|
|||||||
#include <cmath>
|
#include <cmath>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <ctime>
|
#include <ctime>
|
||||||
|
#include <cstdlib>
|
||||||
#include <cstdio>
|
#include <cstdio>
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
@@ -1883,9 +1884,23 @@ namespace StandaloneStreamBenchmark
|
|||||||
bool bProfilePassageComposer=false,bool bAllowUniformSkipsWhenHashing=false,
|
bool bProfilePassageComposer=false,bool bAllowUniformSkipsWhenHashing=false,
|
||||||
bool bLinux77Profile=false,int32 SchedulerPolicy=-1,
|
bool bLinux77Profile=false,int32 SchedulerPolicy=-1,
|
||||||
bool bMovingCameraTrace=false,double MovingCameraSpeedMetersPerSecond=0.0,
|
bool bMovingCameraTrace=false,double MovingCameraSpeedMetersPerSecond=0.0,
|
||||||
double MovingCameraDurationSeconds=12.0,double MovingCameraDwellSeconds=5.0)
|
double MovingCameraDurationSeconds=12.0,double MovingCameraDwellSeconds=5.0,
|
||||||
|
bool bColdWorkerTimeline=false)
|
||||||
{
|
{
|
||||||
using Clock=std::chrono::steady_clock;
|
using Clock=std::chrono::steady_clock;
|
||||||
|
const std::clock_t ColdCpuOrigin=std::clock();
|
||||||
|
const bool bColdTimeline=bColdWorkerTimeline||bInstrumentTileDrain;
|
||||||
|
const auto EmitColdTimeline=[&](const char* Event,const char* Edge,Clock::time_point At)
|
||||||
|
{
|
||||||
|
if(!bColdTimeline)return;
|
||||||
|
const double CpuMilliseconds=1000.0*static_cast<double>(
|
||||||
|
std::clock()-ColdCpuOrigin)/CLOCKS_PER_SEC;
|
||||||
|
std::printf("[ColdTimeline] event=%s edge=%s at_ms=%.3f cpu_ms=%.3f\n",Event,Edge,
|
||||||
|
std::chrono::duration<double,std::milli>(At-ColdStart).count(),CpuMilliseconds);
|
||||||
|
std::fflush(stdout);
|
||||||
|
};
|
||||||
|
const auto MarkColdTimeline=[&](const char* Event,const char* Edge)
|
||||||
|
{EmitColdTimeline(Event,Edge,Clock::now());};
|
||||||
FStandaloneOwnerWorld& World=*OwnerWorld;
|
FStandaloneOwnerWorld& World=*OwnerWorld;
|
||||||
const VoxelForgeCore::WorldAssemblyManifest* Assembly=CoreSnapshot.worldAssembly();
|
const VoxelForgeCore::WorldAssemblyManifest* Assembly=CoreSnapshot.worldAssembly();
|
||||||
std::array<double,9> ClosestDisplayDistances{};
|
std::array<double,9> ClosestDisplayDistances{};
|
||||||
@@ -1913,6 +1928,8 @@ namespace StandaloneStreamBenchmark
|
|||||||
if(MaxLevel>=0)
|
if(MaxLevel>=0)
|
||||||
Desired.Tiles.erase(std::remove_if(Desired.Tiles.begin(),Desired.Tiles.end(),
|
Desired.Tiles.erase(std::remove_if(Desired.Tiles.begin(),Desired.Tiles.end(),
|
||||||
[&](const FVoxelTileKey& Tile){return Tile.Level>MaxLevel;}),Desired.Tiles.end());
|
[&](const FVoxelTileKey& Tile){return Tile.Level>MaxLevel;}),Desired.Tiles.end());
|
||||||
|
const Clock::time_point DesiredRequestAt=Clock::now();
|
||||||
|
EmitColdTimeline("desired_tile_set","requested",DesiredRequestAt);
|
||||||
const bool bHasCameraStep=std::abs(CameraStepVoxels.X)+std::abs(CameraStepVoxels.Y)
|
const bool bHasCameraStep=std::abs(CameraStepVoxels.X)+std::abs(CameraStepVoxels.Y)
|
||||||
+std::abs(CameraStepVoxels.Z)>1.0e-6f;
|
+std::abs(CameraStepVoxels.Z)>1.0e-6f;
|
||||||
FStandaloneDesiredTileSet MovedDesired;
|
FStandaloneDesiredTileSet MovedDesired;
|
||||||
@@ -2049,9 +2066,15 @@ namespace StandaloneStreamBenchmark
|
|||||||
ShapeRequest.detail=DetailVariant;ShapeRequest.includeFarWorld=bIncludeFarWorld&&!bDeferFarWorld;
|
ShapeRequest.detail=DetailVariant;ShapeRequest.includeFarWorld=bIncludeFarWorld&&!bDeferFarWorld;
|
||||||
ShapeRequest.roundTunnels=bRoundTunnels;
|
ShapeRequest.roundTunnels=bRoundTunnels;
|
||||||
ShapeRequest.enhancedTunnelWalls=bEnhancedTunnelWalls;
|
ShapeRequest.enhancedTunnelWalls=bEnhancedTunnelWalls;
|
||||||
ShapeRequest.validatePassageRoutes=!bCaptureSkipWalkability;
|
const char* L86DisablePassages=std::getenv(
|
||||||
|
"VOXELFORGE_L86_DISABLE_PASSAGE_COMPOSITION");
|
||||||
|
const bool bL86DisablePassages=L86DisablePassages
|
||||||
|
&&L86DisablePassages[0]=='1'&&L86DisablePassages[1]=='\0';
|
||||||
|
ShapeRequest.validatePassageRoutes=!bCaptureSkipWalkability&&!bL86DisablePassages;
|
||||||
if(bCaptureSkipWalkability)
|
if(bCaptureSkipWalkability)
|
||||||
std::printf("[CaptureWalkabilityGate] validation=deferred geometry_input=unchanged mode=mesh-export\n");
|
std::printf("[CaptureWalkabilityGate] validation=deferred geometry_input=unchanged mode=mesh-export\n");
|
||||||
|
if(bL86DisablePassages)
|
||||||
|
std::printf("[L86PassageCeiling] validation=skipped geometry=passage_composition_disabled diagnostic_only=true\n");
|
||||||
FStandaloneStreamingGenerator Streamer(World,WorkerCount,bHashDensity,nullptr,MaxLevel,
|
FStandaloneStreamingGenerator Streamer(World,WorkerCount,bHashDensity,nullptr,MaxLevel,
|
||||||
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors,&Shape,bProfileColumnOverlap,
|
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors,&Shape,bProfileColumnOverlap,
|
||||||
FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache,
|
FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache,
|
||||||
@@ -2067,9 +2090,14 @@ namespace StandaloneStreamBenchmark
|
|||||||
Streamer.SetProofBlockCells(ProofBlockCells);
|
Streamer.SetProofBlockCells(ProofBlockCells);
|
||||||
if(bScreenSpaceGate||bReportLodWorkProfile||bInstrumentTileDrain||bLinux77Profile)
|
if(bScreenSpaceGate||bReportLodWorkProfile||bInstrumentTileDrain||bLinux77Profile)
|
||||||
Streamer.SetMesherStageProfilingEnabled(true);
|
Streamer.SetMesherStageProfilingEnabled(true);
|
||||||
if(bInstrumentTileDrain)
|
if(bColdTimeline)
|
||||||
{
|
{
|
||||||
Streamer.EnableColdTimelineTrace(ColdStart,nullptr);
|
Streamer.EnableColdTimelineTrace(ColdStart,nullptr);
|
||||||
|
Streamer.RecordColdTimelineRequests(Desired.Tiles,DesiredRequestAt);
|
||||||
|
EmitColdTimeline("desired_tile_set","request_recorded",Clock::now());
|
||||||
|
}
|
||||||
|
if(bInstrumentTileDrain)
|
||||||
|
{
|
||||||
std::printf("[CpuTileInstrumentation] enabled=true labeled=true flush=after_run proof_block_cells=%d active_lane_batching=%s midpoint_density_memo=%s parallel_sign_edge_repair=%s\n",
|
std::printf("[CpuTileInstrumentation] enabled=true labeled=true flush=after_run proof_block_cells=%d active_lane_batching=%s midpoint_density_memo=%s parallel_sign_edge_repair=%s\n",
|
||||||
ProofBlockCells,bActiveLaneBatching?"on":"off",
|
ProofBlockCells,bActiveLaneBatching?"on":"off",
|
||||||
bExactMidpointDensityMemo?"on":"off",
|
bExactMidpointDensityMemo?"on":"off",
|
||||||
@@ -2077,13 +2105,17 @@ namespace StandaloneStreamBenchmark
|
|||||||
}
|
}
|
||||||
if(bLinux77Profile)VoxelForgeCore::Linux77Profiler::Start(1000);
|
if(bLinux77Profile)VoxelForgeCore::Linux77Profiler::Start(1000);
|
||||||
const auto StreamStart=Clock::now();
|
const auto StreamStart=Clock::now();
|
||||||
|
MarkColdTimeline("initial_shape_free_admission","begin");
|
||||||
const FStandaloneStreamingUpdate EarlyUpdate=Streamer.StartWithDesired(ShapeFreeDesired);
|
const FStandaloneStreamingUpdate EarlyUpdate=Streamer.StartWithDesired(ShapeFreeDesired);
|
||||||
|
MarkColdTimeline("initial_shape_free_admission","end");
|
||||||
if(EarlyUpdate.DesiredTiles!=static_cast<int32>(ShapeFreeDesired.Tiles.size()))
|
if(EarlyUpdate.DesiredTiles!=static_cast<int32>(ShapeFreeDesired.Tiles.size()))
|
||||||
{
|
{
|
||||||
Streamer.Stop();
|
Streamer.Stop();
|
||||||
std::fprintf(stderr,"[StreamPipeline] expected %zu shape-free tiles but queued %d.\n",
|
std::fprintf(stderr,"[StreamPipeline] expected %zu shape-free tiles but queued %d.\n",
|
||||||
ShapeFreeDesired.Tiles.size(),EarlyUpdate.DesiredTiles);return 7;
|
ShapeFreeDesired.Tiles.size(),EarlyUpdate.DesiredTiles);return 7;
|
||||||
}
|
}
|
||||||
|
MarkColdTimeline("shape_compile","begin");
|
||||||
|
MarkColdTimeline("base_compile","begin");
|
||||||
const auto ShapeBuildStart=Clock::now();
|
const auto ShapeBuildStart=Clock::now();
|
||||||
auto ShapeBuildFuture=std::async(std::launch::async,
|
auto ShapeBuildFuture=std::async(std::launch::async,
|
||||||
[&Shape,&Scheduler,ShapeRequest,DiagnosticSlowShapeCompileMs]() mutable
|
[&Shape,&Scheduler,ShapeRequest,DiagnosticSlowShapeCompileMs]() mutable
|
||||||
@@ -2095,6 +2127,8 @@ namespace StandaloneStreamBenchmark
|
|||||||
return std::pair<bool,std::string>{bCompiled,std::move(Error)};
|
return std::pair<bool,std::string>{bCompiled,std::move(Error)};
|
||||||
});
|
});
|
||||||
const auto [bShapeCompiled,BuildError]=ShapeBuildFuture.get();
|
const auto [bShapeCompiled,BuildError]=ShapeBuildFuture.get();
|
||||||
|
MarkColdTimeline("base_compile","end");
|
||||||
|
MarkColdTimeline("shape_compile","end");
|
||||||
if(!bShapeCompiled)
|
if(!bShapeCompiled)
|
||||||
{
|
{
|
||||||
Streamer.Stop();
|
Streamer.Stop();
|
||||||
@@ -4994,8 +5028,10 @@ namespace StandaloneStreamBenchmark
|
|||||||
const int32 EarlyDesiredCount=static_cast<int32>(EarlyDesired.Tiles.size());
|
const int32 EarlyDesiredCount=static_cast<int32>(EarlyDesired.Tiles.size());
|
||||||
const int32 InitialDesiredCount=bDeferFarWorld?EarlyDesiredCount:
|
const int32 InitialDesiredCount=bDeferFarWorld?EarlyDesiredCount:
|
||||||
static_cast<int32>(Desired.Tiles.size());
|
static_cast<int32>(Desired.Tiles.size());
|
||||||
|
MarkColdTimeline("lod0_1_admission","begin");
|
||||||
FStandaloneStreamingUpdate Update=Streamer.UpdateDesiredWithSet(
|
FStandaloneStreamingUpdate Update=Streamer.UpdateDesiredWithSet(
|
||||||
bDeferFarWorld?EarlyDesired:Desired,{},0);
|
bDeferFarWorld?EarlyDesired:Desired,{},0);
|
||||||
|
MarkColdTimeline("lod0_1_admission","end");
|
||||||
if(Update.DesiredTiles!=InitialDesiredCount)
|
if(Update.DesiredTiles!=InitialDesiredCount)
|
||||||
{
|
{
|
||||||
Streamer.Stop();
|
Streamer.Stop();
|
||||||
@@ -5003,7 +5039,9 @@ namespace StandaloneStreamBenchmark
|
|||||||
InitialDesiredCount,Update.DesiredTiles);return 7;
|
InitialDesiredCount,Update.DesiredTiles);return 7;
|
||||||
}
|
}
|
||||||
double DeferredFarShapeSeconds=0.0;
|
double DeferredFarShapeSeconds=0.0;
|
||||||
|
Clock::time_point EarlyTilesDoneAt{};
|
||||||
bool bDeferredFarTilesQueued=false;
|
bool bDeferredFarTilesQueued=false;
|
||||||
|
bool bLOD01DoneMarked=false;
|
||||||
if(bDeferFarWorld)
|
if(bDeferFarWorld)
|
||||||
std::printf("[ShapeSchedule] defer_far_parents=1 early_tiles=%d late_tiles=%zu start=after_base_shape_compile\n",
|
std::printf("[ShapeSchedule] defer_far_parents=1 early_tiles=%d late_tiles=%zu start=after_base_shape_compile\n",
|
||||||
EarlyDesiredCount,Desired.Tiles.size()-EarlyDesired.Tiles.size());
|
EarlyDesiredCount,Desired.Tiles.size()-EarlyDesired.Tiles.size());
|
||||||
@@ -5242,9 +5280,10 @@ namespace StandaloneStreamBenchmark
|
|||||||
*VoxelForgeCore::Linux77Profiler::RouteCount*2>,9> L77DispatchByLOD{};
|
*VoxelForgeCore::Linux77Profiler::RouteCount*2>,9> L77DispatchByLOD{};
|
||||||
std::array<std::uint64_t,9> L77GridInteriorByLOD{},L77GridHaloByLOD{};
|
std::array<std::uint64_t,9> L77GridInteriorByLOD{},L77GridHaloByLOD{};
|
||||||
std::array<double,9> L77LODReadySeconds{};
|
std::array<double,9> L77LODReadySeconds{};
|
||||||
std::array<double,9> LODReadySeconds{};
|
std::array<double,9> LODReadySeconds{},LODPublishedReadySeconds{},LODPipelineReadySeconds{};
|
||||||
std::array<std::uint64_t,9> SkirtTrianglesByLOD{};
|
std::array<std::uint64_t,9> SkirtTrianglesByLOD{};
|
||||||
std::array<std::uint64_t,9> TileCountByLOD{};
|
std::array<std::uint64_t,9> TileCountByLOD{};
|
||||||
|
bool bLOD2DoneMarked=false;
|
||||||
std::array<std::uint64_t,9> ClassifiedAirTilesByLOD{},ClassifiedSolidTilesByLOD{};
|
std::array<std::uint64_t,9> ClassifiedAirTilesByLOD{},ClassifiedSolidTilesByLOD{};
|
||||||
std::array<std::uint64_t,9> ClassifiedMixedTilesByLOD{},UnclassifiedTilesByLOD{};
|
std::array<std::uint64_t,9> ClassifiedMixedTilesByLOD{},UnclassifiedTilesByLOD{};
|
||||||
std::array<std::uint64_t,9> MeshingSkippedTilesByLOD{},DensityGridTilesByLOD{};
|
std::array<std::uint64_t,9> MeshingSkippedTilesByLOD{},DensityGridTilesByLOD{};
|
||||||
@@ -5311,9 +5350,19 @@ namespace StandaloneStreamBenchmark
|
|||||||
const std::size_t LOD=static_cast<std::size_t>(Tile.Key.Level);
|
const std::size_t LOD=static_cast<std::size_t>(Tile.Key.Level);
|
||||||
const auto& TileDiagnostics=Tile.CoreDiagnostics;
|
const auto& TileDiagnostics=Tile.CoreDiagnostics;
|
||||||
++TileCountByLOD[LOD];
|
++TileCountByLOD[LOD];
|
||||||
if(TileCountByLOD[LOD]==ExpectedTilesByLOD[LOD])
|
LODReadySeconds[LOD]=std::max(LODReadySeconds[LOD],
|
||||||
LODReadySeconds[LOD]=std::chrono::duration<double>(
|
std::chrono::duration<double>(Tile.CompletedAt-ColdStart).count());
|
||||||
Tile.CompletedAt-ColdStart).count();
|
LODPublishedReadySeconds[LOD]=std::max(LODPublishedReadySeconds[LOD],
|
||||||
|
std::chrono::duration<double>(Tile.PublishedAt-ColdStart).count());
|
||||||
|
LODPipelineReadySeconds[LOD]=std::max(LODPipelineReadySeconds[LOD],
|
||||||
|
std::chrono::duration<double>(Tile.DrainedAt-ColdStart).count());
|
||||||
|
if(LOD==2&&!bLOD2DoneMarked
|
||||||
|
&&TileCountByLOD[LOD]==ExpectedTilesByLOD[LOD])
|
||||||
|
{
|
||||||
|
MarkColdTimeline("lod2_done","pipeline_drain");
|
||||||
|
MarkColdTimeline("lod0_2_done","pipeline_drain");
|
||||||
|
bLOD2DoneMarked=true;
|
||||||
|
}
|
||||||
if(bLinux77Profile)
|
if(bLinux77Profile)
|
||||||
{
|
{
|
||||||
const auto& D=TileDiagnostics;
|
const auto& D=TileDiagnostics;
|
||||||
@@ -5518,17 +5567,32 @@ namespace StandaloneStreamBenchmark
|
|||||||
FullDetailFinished,FullDetailExpected);
|
FullDetailFinished,FullDetailExpected);
|
||||||
bFullDetailReported=true;
|
bFullDetailReported=true;
|
||||||
}
|
}
|
||||||
|
if(!bLOD01DoneMarked&&Finished==EarlyDesiredCount)
|
||||||
|
{
|
||||||
|
EarlyTilesDoneAt=Clock::now();
|
||||||
|
EmitColdTimeline("lod0_1_done","publish_drained",EarlyTilesDoneAt);
|
||||||
|
bLOD01DoneMarked=true;
|
||||||
|
}
|
||||||
if(bDeferFarWorld&&!bDeferredFarTilesQueued&&Finished==EarlyDesiredCount
|
if(bDeferFarWorld&&!bDeferredFarTilesQueued&&Finished==EarlyDesiredCount
|
||||||
&&Desired.Tiles.size()>EarlyDesired.Tiles.size())
|
&&Desired.Tiles.size()>EarlyDesired.Tiles.size())
|
||||||
{
|
{
|
||||||
|
MarkColdTimeline("far_parent_build","begin");
|
||||||
const auto FarBuildStart=Clock::now();
|
const auto FarBuildStart=Clock::now();
|
||||||
Shape.buildFarWorld();
|
if(!Shape.buildFarWorld())
|
||||||
|
{
|
||||||
|
Streamer.Stop();
|
||||||
|
std::fprintf(stderr,"[StreamPipeline] deferred far-world build failed.\n");
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
DeferredFarShapeSeconds=std::chrono::duration<double>(
|
DeferredFarShapeSeconds=std::chrono::duration<double>(
|
||||||
Clock::now()-FarBuildStart).count();
|
Clock::now()-FarBuildStart).count();
|
||||||
|
MarkColdTimeline("far_parent_build","end");
|
||||||
std::printf("[ShapeSchedule] defer_far_parents=1 early_ready_s=%.6f far_shape_s=%.6f action=publish_before_lod2_tiles\n",
|
std::printf("[ShapeSchedule] defer_far_parents=1 early_ready_s=%.6f far_shape_s=%.6f action=publish_before_lod2_tiles\n",
|
||||||
std::chrono::duration<double>(Clock::now()-ColdStart).count(),
|
std::chrono::duration<double>(Clock::now()-ColdStart).count(),
|
||||||
DeferredFarShapeSeconds);
|
DeferredFarShapeSeconds);
|
||||||
|
MarkColdTimeline("lod2_queue","begin");
|
||||||
Update=Streamer.UpdateDesiredWithSet(Desired,EarlyDesired.Tiles,0);
|
Update=Streamer.UpdateDesiredWithSet(Desired,EarlyDesired.Tiles,0);
|
||||||
|
MarkColdTimeline("lod2_queue","end");
|
||||||
if(Update.DesiredTiles!=static_cast<int32>(Desired.Tiles.size()))
|
if(Update.DesiredTiles!=static_cast<int32>(Desired.Tiles.size()))
|
||||||
{
|
{
|
||||||
Streamer.Stop();
|
Streamer.Stop();
|
||||||
@@ -5764,9 +5828,27 @@ namespace StandaloneStreamBenchmark
|
|||||||
if(ExpectedTilesByLOD[LOD]!=0)
|
if(ExpectedTilesByLOD[LOD]!=0)
|
||||||
NearLODReadySeconds=std::max(NearLODReadySeconds,LODReadySeconds[LOD]);
|
NearLODReadySeconds=std::max(NearLODReadySeconds,LODReadySeconds[LOD]);
|
||||||
}
|
}
|
||||||
|
double NearLODPublishedSeconds=0.0,NearLODPipelineSeconds=0.0;
|
||||||
|
for(std::size_t LOD=0;LOD<=2;++LOD)
|
||||||
|
{
|
||||||
|
if(ExpectedTilesByLOD[LOD]==0)continue;
|
||||||
|
NearLODPublishedSeconds=std::max(NearLODPublishedSeconds,
|
||||||
|
LODPublishedReadySeconds[LOD]);
|
||||||
|
NearLODPipelineSeconds=std::max(NearLODPipelineSeconds,
|
||||||
|
LODPipelineReadySeconds[LOD]);
|
||||||
|
std::printf("[WorldAssemblerLODPipelineReady] lod=%zu generated_s=%.6f scheduler_published_s=%.6f pipeline_drained_s=%.6f\n",
|
||||||
|
LOD,LODReadySeconds[LOD],LODPublishedReadySeconds[LOD],
|
||||||
|
LODPipelineReadySeconds[LOD]);
|
||||||
|
}
|
||||||
std::printf("[WorldAssemblerNearLODReady] lods=0-2 tiles=%llu/%llu cold_ready_s=%.6f\n",
|
std::printf("[WorldAssemblerNearLODReady] lods=0-2 tiles=%llu/%llu cold_ready_s=%.6f\n",
|
||||||
static_cast<unsigned long long>(NearLODCompleted),
|
static_cast<unsigned long long>(NearLODCompleted),
|
||||||
static_cast<unsigned long long>(NearLODExpected),NearLODReadySeconds);
|
static_cast<unsigned long long>(NearLODExpected),NearLODReadySeconds);
|
||||||
|
std::printf("[WorldAssemblerNearLODPublished] lods=0-2 tiles=%llu/%llu cold_ready_s=%.6f\n",
|
||||||
|
static_cast<unsigned long long>(NearLODCompleted),
|
||||||
|
static_cast<unsigned long long>(NearLODExpected),NearLODPublishedSeconds);
|
||||||
|
std::printf("[WorldAssemblerNearLODPipelineReady] lods=0-2 tiles=%llu/%llu cold_ready_s=%.6f\n",
|
||||||
|
static_cast<unsigned long long>(NearLODCompleted),
|
||||||
|
static_cast<unsigned long long>(NearLODExpected),NearLODPipelineSeconds);
|
||||||
for(std::size_t LOD=0;LOD<TileCountByLOD.size();++LOD)
|
for(std::size_t LOD=0;LOD<TileCountByLOD.size();++LOD)
|
||||||
if(ExpectedTilesByLOD[LOD]!=0)
|
if(ExpectedTilesByLOD[LOD]!=0)
|
||||||
std::printf("[WorldAssemblerLODReady] lod=%zu tiles=%llu cold_ready_s=%.6f\n",
|
std::printf("[WorldAssemblerLODReady] lod=%zu tiles=%llu cold_ready_s=%.6f\n",
|
||||||
|
|||||||
@@ -54,6 +54,8 @@ struct FStandaloneCpuTile
|
|||||||
std::chrono::steady_clock::time_point QueuedAt{};
|
std::chrono::steady_clock::time_point QueuedAt{};
|
||||||
std::chrono::steady_clock::time_point StartedAt{};
|
std::chrono::steady_clock::time_point StartedAt{};
|
||||||
std::chrono::steady_clock::time_point CompletedAt{};
|
std::chrono::steady_clock::time_point CompletedAt{};
|
||||||
|
std::chrono::steady_clock::time_point PublishedAt{};
|
||||||
|
std::chrono::steady_clock::time_point DrainedAt{};
|
||||||
uint64 DensityHash=14695981039346656037ull;
|
uint64 DensityHash=14695981039346656037ull;
|
||||||
bool bHasDensityHash=false;
|
bool bHasDensityHash=false;
|
||||||
uint64 MeshHash=14695981039346656037ull;
|
uint64 MeshHash=14695981039346656037ull;
|
||||||
@@ -362,8 +364,12 @@ struct FStandaloneTileJob
|
|||||||
|
|
||||||
struct FStandaloneCpuTileTimelineEvent
|
struct FStandaloneCpuTileTimelineEvent
|
||||||
{
|
{
|
||||||
enum class EType : uint8 { Queued, Dispatch, Finish };
|
enum class EType : uint8
|
||||||
EType Type=EType::Queued;
|
{
|
||||||
|
Requested, Admitted, Dispatch, Finish, Published, Drained,
|
||||||
|
WorkerStart, WorkerIdleBegin, WorkerIdleEnd, WorkerStop
|
||||||
|
};
|
||||||
|
EType Type=EType::Admitted;
|
||||||
FVoxelTileKey Key;
|
FVoxelTileKey Key;
|
||||||
int32 WorkerId=-1;
|
int32 WorkerId=-1;
|
||||||
size_t QueueDepth=0;
|
size_t QueueDepth=0;
|
||||||
@@ -517,6 +523,15 @@ public:
|
|||||||
bColdTimelineTrace=true;
|
bColdTimelineTrace=true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RecordColdTimelineRequests(const std::vector<FVoxelTileKey>& Keys,
|
||||||
|
std::chrono::steady_clock::time_point RequestedAt)
|
||||||
|
{
|
||||||
|
if(!bColdTimelineTrace)return;
|
||||||
|
for(const FVoxelTileKey& Key:Keys)
|
||||||
|
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Requested,Key,-1,
|
||||||
|
0,0,0,RequestedAt,RequestedAt);
|
||||||
|
}
|
||||||
|
|
||||||
void DumpColdTimelineTrace() const
|
void DumpColdTimelineTrace() const
|
||||||
{
|
{
|
||||||
if(!bColdTimelineTrace)return;
|
if(!bColdTimelineTrace)return;
|
||||||
@@ -532,16 +547,14 @@ public:
|
|||||||
Events.size(),WorkerCount);
|
Events.size(),WorkerCount);
|
||||||
for(const auto& Event:Events)
|
for(const auto& Event:Events)
|
||||||
{
|
{
|
||||||
const char* Type=Event.Type==FStandaloneCpuTileTimelineEvent::EType::Queued
|
const char* Type=TimelineEventName(Event.Type);
|
||||||
?"queued":Event.Type==FStandaloneCpuTileTimelineEvent::EType::Dispatch
|
|
||||||
?"dispatch":"finish";
|
|
||||||
std::printf("[CpuTileTimeline] event=%s key=(%d,%d,%d,lod%d) at_ms=%.3f frame=%llu worker_id=%d queue_depth=%zu active_workers=%zu priority_rank=%zu queued_ms=%.3f dispatch_ms=%.3f finish_ms=%.3f service_ms=%.3f worker_ms=%.3f\n",
|
std::printf("[CpuTileTimeline] event=%s key=(%d,%d,%d,lod%d) at_ms=%.3f frame=%llu worker_id=%d queue_depth=%zu active_workers=%zu priority_rank=%zu queued_ms=%.3f dispatch_ms=%.3f finish_ms=%.3f service_ms=%.3f worker_ms=%.3f\n",
|
||||||
Type,Event.Key.Coord.X,Event.Key.Coord.Y,Event.Key.Coord.Z,Event.Key.Level,
|
Type,Event.Key.Coord.X,Event.Key.Coord.Y,Event.Key.Coord.Z,Event.Key.Level,
|
||||||
TimelineMilliseconds(Event.At),static_cast<unsigned long long>(TimelineFrame()),
|
TimelineMilliseconds(Event.At),static_cast<unsigned long long>(TimelineFrame()),
|
||||||
Event.WorkerId,Event.QueueDepth,Event.ActiveWorkers,Event.PriorityRank,
|
Event.WorkerId,Event.QueueDepth,Event.ActiveWorkers,Event.PriorityRank,
|
||||||
TimelineMilliseconds(Event.QueuedAt),TimelineMilliseconds(Event.StartedAt),
|
TimelineMilliseconds(Event.QueuedAt),TimelineMilliseconds(Event.StartedAt),
|
||||||
TimelineMilliseconds(Event.FinishedAt),
|
TimelineMilliseconds(Event.FinishedAt),
|
||||||
Event.StartedAt.time_since_epoch().count()!=0
|
Event.Type==FStandaloneCpuTileTimelineEvent::EType::Finish
|
||||||
?std::chrono::duration<double,std::milli>(Event.At-Event.StartedAt).count():-1.0,
|
?std::chrono::duration<double,std::milli>(Event.At-Event.StartedAt).count():-1.0,
|
||||||
Event.WorkerMilliseconds);
|
Event.WorkerMilliseconds);
|
||||||
if(Event.Type==FStandaloneCpuTileTimelineEvent::EType::Finish)
|
if(Event.Type==FStandaloneCpuTileTimelineEvent::EType::Finish)
|
||||||
@@ -747,7 +760,16 @@ public:
|
|||||||
std::vector<FStandaloneCpuTile> Out;
|
std::vector<FStandaloneCpuTile> Out;
|
||||||
std::lock_guard<std::mutex> Lock(Mutex);
|
std::lock_guard<std::mutex> Lock(Mutex);
|
||||||
Out.swap(Ready);
|
Out.swap(Ready);
|
||||||
for(const FStandaloneCpuTile& Tile:Out)Outstanding.erase(Tile.Key);
|
const FClock::time_point DrainedAt=FClock::now();
|
||||||
|
for(FStandaloneCpuTile& Tile:Out)
|
||||||
|
{
|
||||||
|
Tile.DrainedAt=DrainedAt;
|
||||||
|
Outstanding.erase(Tile.Key);
|
||||||
|
if(bColdTimelineTrace)
|
||||||
|
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Drained,
|
||||||
|
Tile.Key,-1,Queue.size(),Running.size(),0,DrainedAt,
|
||||||
|
Tile.QueuedAt,Tile.StartedAt,Tile.CompletedAt,Tile.WorkerSeconds*1000.0);
|
||||||
|
}
|
||||||
return Out;
|
return Out;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1083,7 +1105,7 @@ private:
|
|||||||
Job.QueuedAt=FClock::now();
|
Job.QueuedAt=FClock::now();
|
||||||
Queue.push_back(Job);
|
Queue.push_back(Job);
|
||||||
if(bColdTimelineTrace)
|
if(bColdTimelineTrace)
|
||||||
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Queued,Key,-1,
|
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Admitted,Key,-1,
|
||||||
Queue.size(),Running.size(),Job.PriorityRank,Job.QueuedAt,Job.QueuedAt);
|
Queue.size(),Running.size(),Job.PriorityRank,Job.QueuedAt,Job.QueuedAt);
|
||||||
Outstanding.insert(Key);++Stats.NewJobs;
|
Outstanding.insert(Key);++Stats.NewJobs;
|
||||||
if(Key.Level<=2)++NearOutstandingJobs;
|
if(Key.Level<=2)++NearOutstandingJobs;
|
||||||
@@ -1278,6 +1300,9 @@ private:
|
|||||||
|
|
||||||
void WorkerLoop(int32 WorkerId)
|
void WorkerLoop(int32 WorkerId)
|
||||||
{
|
{
|
||||||
|
if(bColdTimelineTrace)
|
||||||
|
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::WorkerStart,{},
|
||||||
|
WorkerId,0,0,0,FClock::now());
|
||||||
constexpr uint32 PriorityTaskBurst=3;
|
constexpr uint32 PriorityTaskBurst=3;
|
||||||
uint32 ConsecutivePriorityTasks=0;
|
uint32 ConsecutivePriorityTasks=0;
|
||||||
for(;;)
|
for(;;)
|
||||||
@@ -1295,8 +1320,21 @@ private:
|
|||||||
return !Queue.empty()&&((SchedulerPolicy!=1&&!bStationaryNearGate)
|
return !Queue.empty()&&((SchedulerPolicy!=1&&!bStationaryNearGate)
|
||||||
||NearOutstandingJobs==0||Queue.front().Key.Level<=2);
|
||NearOutstandingJobs==0||Queue.front().Key.Level<=2);
|
||||||
};
|
};
|
||||||
|
const bool bEnteringIdle=!bStop&&PriorityTasks.empty()&&!HasEligibleTile();
|
||||||
|
if(bEnteringIdle&&bColdTimelineTrace)
|
||||||
|
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::WorkerIdleBegin,
|
||||||
|
{},WorkerId,Queue.size(),Running.size(),0,FClock::now());
|
||||||
Wake.wait(Lock,[&](){return bStop||!PriorityTasks.empty()||HasEligibleTile();});
|
Wake.wait(Lock,[&](){return bStop||!PriorityTasks.empty()||HasEligibleTile();});
|
||||||
if(bStop)return;
|
if(bEnteringIdle&&bColdTimelineTrace)
|
||||||
|
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::WorkerIdleEnd,
|
||||||
|
{},WorkerId,Queue.size(),Running.size(),0,FClock::now());
|
||||||
|
if(bStop)
|
||||||
|
{
|
||||||
|
if(bColdTimelineTrace)
|
||||||
|
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::WorkerStop,{},
|
||||||
|
WorkerId,Queue.size(),Running.size(),0,FClock::now());
|
||||||
|
return;
|
||||||
|
}
|
||||||
const bool bTileEligible=HasEligibleTile();
|
const bool bTileEligible=HasEligibleTile();
|
||||||
const bool bPreferPriority=!PriorityTasks.empty()
|
const bool bPreferPriority=!PriorityTasks.empty()
|
||||||
&&(!bTileEligible||ConsecutivePriorityTasks<PriorityTaskBurst);
|
&&(!bTileEligible||ConsecutivePriorityTasks<PriorityTaskBurst);
|
||||||
@@ -1335,7 +1373,20 @@ private:
|
|||||||
Tile.QueuedAt,Tile.StartedAt,Tile.CompletedAt,Tile.WorkerSeconds*1000.0,
|
Tile.QueuedAt,Tile.StartedAt,Tile.CompletedAt,Tile.WorkerSeconds*1000.0,
|
||||||
&Tile.CoreDiagnostics,Tile.DensitySamples,Tile.SkirtTriangleCount,
|
&Tile.CoreDiagnostics,Tile.DensitySamples,Tile.SkirtTriangleCount,
|
||||||
Tile.PayloadBuildMilliseconds);
|
Tile.PayloadBuildMilliseconds);
|
||||||
|
const FVoxelTileKey PublishedKey=Tile.Key;
|
||||||
|
const auto PublishedQueuedAt=Tile.QueuedAt;
|
||||||
|
const auto PublishedStartedAt=Tile.StartedAt;
|
||||||
|
const auto PublishedCompletedAt=Tile.CompletedAt;
|
||||||
|
const double PublishedWorkerMs=Tile.WorkerSeconds*1000.0;
|
||||||
|
const int32 PublishedWorkerId=Tile.WorkerId;
|
||||||
Ready.push_back(std::move(Tile));
|
Ready.push_back(std::move(Tile));
|
||||||
|
const FClock::time_point PublishedAt=FClock::now();
|
||||||
|
Ready.back().PublishedAt=PublishedAt;
|
||||||
|
if(bColdTimelineTrace)
|
||||||
|
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Published,
|
||||||
|
PublishedKey,PublishedWorkerId,Queue.size(),Running.size(),Job.PriorityRank,
|
||||||
|
PublishedAt,PublishedQueuedAt,PublishedStartedAt,PublishedCompletedAt,
|
||||||
|
PublishedWorkerMs);
|
||||||
}
|
}
|
||||||
Wake.notify_all();
|
Wake.notify_all();
|
||||||
}
|
}
|
||||||
@@ -1351,6 +1402,24 @@ private:
|
|||||||
{
|
{
|
||||||
return TimelineFrameOrdinal?TimelineFrameOrdinal->load(std::memory_order_relaxed):0;
|
return TimelineFrameOrdinal?TimelineFrameOrdinal->load(std::memory_order_relaxed):0;
|
||||||
}
|
}
|
||||||
|
static const char* TimelineEventName(FStandaloneCpuTileTimelineEvent::EType Type)
|
||||||
|
{
|
||||||
|
using EType=FStandaloneCpuTileTimelineEvent::EType;
|
||||||
|
switch(Type)
|
||||||
|
{
|
||||||
|
case EType::Requested:return "requested";
|
||||||
|
case EType::Admitted:return "admitted";
|
||||||
|
case EType::Dispatch:return "started";
|
||||||
|
case EType::Finish:return "finished";
|
||||||
|
case EType::Published:return "published";
|
||||||
|
case EType::Drained:return "drained";
|
||||||
|
case EType::WorkerStart:return "worker_start";
|
||||||
|
case EType::WorkerIdleBegin:return "idle_begin";
|
||||||
|
case EType::WorkerIdleEnd:return "idle_end";
|
||||||
|
case EType::WorkerStop:return "worker_stop";
|
||||||
|
}
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
std::size_t PassageOverlapScore(const FVoxelTileKey& Key) const
|
std::size_t PassageOverlapScore(const FVoxelTileKey& Key) const
|
||||||
{
|
{
|
||||||
const FIntVector Origin=Key.OriginVoxels();
|
const FIntVector Origin=Key.OriginVoxels();
|
||||||
|
|||||||
Reference in new issue
Block a user