research: arrival/far streaming study (L120, no production change)
Far holes coexist with idle workers: far tiles wait median 1.12 s / p95 3.21 s from worker completion to apply while generating in p95 10 ms, so the game-thread result queue and admission are the limit. Priority split failed; 10 slots helped but broke the worst-tick bar. LOD2 still waits for far parents. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed
1 parent
95cf8de107
commit
161565936c
5 files changed
+1025
No files matched your search
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Five-run interleaved L120 priority-split comparison at 30 m/s."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
HOST = Path("/home/jahni/VFHost")
|
||||
EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd")
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
HEADLESS = ROOT / "Saved/Linux/l108/bin/VoxelForgeCoreHeadless-l108-after-clang"
|
||||
OUT = ROOT / "Saved/Linux/l120/movement/priority-split-verified"
|
||||
HOST_SAVED = HOST / "Saved/Linux/l120"
|
||||
HOST_OUT = HOST / "Plugins/VoxelForge/Saved/Linux/l120/priority-split-verified"
|
||||
ACTIVE_DESIGN = "priority_split"
|
||||
SPAWN_M = (19.166015625, -14.2001953125, -753.3740234375)
|
||||
MOVING_SAMPLE = re.compile(r"^\[L79MovingSample\] (.*)$", re.M)
|
||||
MOVING_SUMMARY = re.compile(r"^\[L79MovingSummary\] (.*)$", re.M)
|
||||
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
|
||||
|
||||
def p99(values: list[float]) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
values.sort()
|
||||
return values[max(0, math.ceil(0.99 * len(values)) - 1)]
|
||||
|
||||
|
||||
def parse_events(path: Path) -> list[dict]:
|
||||
events = []
|
||||
if not path.is_file():
|
||||
return events
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
events.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return events
|
||||
|
||||
|
||||
def summarize(rows: list[dict], phase: str) -> dict:
|
||||
moving = [row for row in rows if row["phase"] == "move"]
|
||||
dwell = [row for row in rows if row["phase"] == "dwell"]
|
||||
def moving_mean(name: str) -> float | None:
|
||||
values = [row[name] for row in moving if name in row]
|
||||
return statistics.mean(values) if values else None
|
||||
complete = next((row for row in dwell
|
||||
if row["near_ready_pct"] >= 99.999
|
||||
and row["far_hole_pct"] <= 0.001), None)
|
||||
end = dwell[-1] if dwell else {}
|
||||
return {
|
||||
"moving_samples": len(moving),
|
||||
"dwell_samples": len(dwell),
|
||||
"near_ready_moving_mean_pct": moving_mean("near_ready_pct"),
|
||||
"lod0_ready_moving_mean_pct": moving_mean("lod0_ready_pct"),
|
||||
"lod1_ready_moving_mean_pct": moving_mean("lod1_ready_pct"),
|
||||
"lod2_ready_moving_mean_pct": moving_mean("lod2_ready_pct"),
|
||||
"far_holes_moving_mean_pct": moving_mean("far_hole_pct"),
|
||||
"near_ready_dwell_end_pct": end.get("near_ready_pct"),
|
||||
"far_holes_dwell_end_pct": end.get("far_hole_pct"),
|
||||
"dwell_complete_after_stop_s": complete.get("since_stop_s") if complete else None,
|
||||
"phase": phase,
|
||||
}
|
||||
|
||||
|
||||
def run_headless(condition: str, repetition: int) -> dict:
|
||||
worker_count = 8 if condition == "baseline" else 10
|
||||
folder = OUT / condition / "headless"
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
log = folder / f"fast-r{repetition}.log"
|
||||
camera = ",".join(f"{value:.12f}" for value in SPAWN_M) + ",0,0"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||
"timeout", "1740s", str(HEADLESS), str(DATA),
|
||||
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
|
||||
f"--workers={worker_count}", "--clip-radius=3", "--max-clip-level=8",
|
||||
"--max-level=8", f"--camera={camera}", "--profile-lod-work",
|
||||
"--stream-scheduler-policy=3", "--moving-camera-speed-mps=30.0",
|
||||
"--moving-camera-duration-s=12", "--moving-camera-dwell-s=5",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as stream:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=stream,
|
||||
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||
content = log.read_text(encoding="utf-8", errors="replace")
|
||||
rows = []
|
||||
for match in MOVING_SAMPLE.finditer(content):
|
||||
row = dict(FIELDS.findall(match.group(1)))
|
||||
for name in ("t_s", "near_ready_pct", "walkable_ring_ready_pct",
|
||||
"far_hole_pct", "queue_update_ms"):
|
||||
if name in row:
|
||||
row[name] = float(row[name])
|
||||
row["phase"] = row.get("phase", "move")
|
||||
row["since_stop_s"] = max(0.0, row.get("t_s", 0.0) - 12.0)
|
||||
rows.append(row)
|
||||
engine_summary = {}
|
||||
if match := MOVING_SUMMARY.search(content):
|
||||
engine_summary = dict(FIELDS.findall(match.group(1)))
|
||||
if result.returncode or not rows or int(engine_summary.get("generation_failures", "-1")) != 0:
|
||||
raise RuntimeError(f"headless run failed: {log}; exit={result.returncode}")
|
||||
moving_updates = [row["queue_update_ms"] for row in rows
|
||||
if row["phase"] == "move" and "queue_update_ms" in row]
|
||||
return {
|
||||
"condition": condition, "platform": "headless", "speed_mps": 30.0,
|
||||
"repetition": repetition, "worker_count": worker_count,
|
||||
"exit_code": result.returncode,
|
||||
"wall_s": round(time.monotonic() - started, 3),
|
||||
"update_work_p99_ms": p99(moving_updates),
|
||||
"update_work_max_ms": max(moving_updates, default=None),
|
||||
"log": str(log.relative_to(ROOT)), **summarize(rows, "headless"),
|
||||
}
|
||||
|
||||
|
||||
def run_unreal(condition: str, repetition: int) -> dict:
|
||||
core_limit = 12
|
||||
if ACTIVE_DESIGN == "priority_split":
|
||||
worker_cap = 1
|
||||
priority_split = 0 if condition == "baseline" else 1
|
||||
else:
|
||||
worker_cap = 1 if condition == "baseline" else 0
|
||||
priority_split = 0
|
||||
folder = OUT / condition / "unreal"
|
||||
host_folder = HOST_OUT / condition / "output"
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
trace = folder / f"fast-r{repetition}.trace.jsonl"
|
||||
abslog = folder / f"fast-r{repetition}.log"
|
||||
stdout = folder / f"fast-r{repetition}.stdout.log"
|
||||
export = host_folder / f"fast-r{repetition}"
|
||||
userdir = HOST_SAVED / "user" / condition / f"fast-r{repetition}"
|
||||
zendata = HOST_SAVED / "zen" / condition / f"fast-r{repetition}"
|
||||
ddc = HOST_SAVED / "DDC"
|
||||
for path in (export, userdir, zendata, ddc):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M)
|
||||
command = [
|
||||
str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended",
|
||||
"-nop4", "-nosplash", "-NoSound", f"-corelimit={core_limit}",
|
||||
"-NoZenAutoLaunch", "-notraceserver", "-nocrashreports",
|
||||
"-DisablePlugins=AndroidFileServer", "-ddc=NoZenLocalFallback",
|
||||
f"-userdir={userdir}", f"-ZenDataPath={zendata}",
|
||||
f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0",
|
||||
"-archetype=TunnelNetwork", "-slot=0", "-modes=export", "-gameconfig=1",
|
||||
"-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25",
|
||||
"-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0",
|
||||
"-coreplantest=0", "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings",
|
||||
"-corehybridmesher=0", f"-out={export}",
|
||||
"-voxel.StreamingCoarseFirst=1", "-voxel.StreamingDynamicNearReserve=1",
|
||||
"-voxel.StreamingLegacyPendingResultCap=0", "-voxel.CoreRegionConcurrentAdmission=1",
|
||||
"-voxel.StreamingResultBacklogCap=256", "-voxel.StreamingReadyQueueDepth=8",
|
||||
f"-voxel.StreamingBackgroundWorkerCap={worker_cap}",
|
||||
"-voxel.CoreColumnCacheDirectMapped=0",
|
||||
f"-voxel.StreamingTilePrioritySplit={priority_split}",
|
||||
"-voxel.CoreMCGradientMemoization=1", "-voxel.StreamingCollisionLookahead=1",
|
||||
"-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=3000.0",
|
||||
"-voxel.TestMoveStartSeconds=6.0", "-voxel.TestMoveDurationSeconds=12.0",
|
||||
"-voxel.TestExitSeconds=23.0", "-voxel.TestMoveDirection=1,0,0",
|
||||
"-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0",
|
||||
"-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}",
|
||||
"-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1",
|
||||
"-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}",
|
||||
f"-abslog={abslog}",
|
||||
]
|
||||
wrapped = ["taskset", "-c", "0-11", "timeout", "1740s", *command]
|
||||
started = time.monotonic()
|
||||
with stdout.open("w", encoding="utf-8") as stream:
|
||||
result = subprocess.run(wrapped, cwd=ROOT, stdout=stream,
|
||||
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||
events = parse_events(trace)
|
||||
samples = [event for event in events if event.get("type") == "streaming_sample"]
|
||||
moving_samples = [event for event in samples
|
||||
if float(event.get("speed_cm_s", 0.0)) >= 50.0]
|
||||
move_start = min((float(event.get("t_s", 0.0)) for event in moving_samples),
|
||||
default=float("inf"))
|
||||
stop_event = next((event for event in events
|
||||
if event.get("type") == "streaming_test_move_stopped"), None)
|
||||
stop = float(stop_event.get("t_s", 0.0)) if stop_event else float("inf")
|
||||
rows = []
|
||||
for event in samples:
|
||||
desired = event.get("desired_by_lod", [0] * 9)
|
||||
resident = event.get("resident_by_lod", [0] * 9)
|
||||
covered = event.get("covered_by_lod", resident)
|
||||
near_desired = sum(desired[:3])
|
||||
near_resident = sum(resident[:3])
|
||||
far_desired = sum(desired[3:])
|
||||
far_covered = sum(covered[3:])
|
||||
t_s = float(event.get("t_s", 0.0))
|
||||
speed = float(event.get("speed_cm_s", 0.0))
|
||||
phase = "move" if speed >= 50.0 else ("dwell" if t_s >= stop else "arrival")
|
||||
rows.append({
|
||||
"phase": phase, "t_s": t_s,
|
||||
"since_stop_s": max(0.0, t_s - stop) if phase == "dwell" else None,
|
||||
"near_ready_pct": 100.0 * near_resident / max(1, near_desired),
|
||||
"lod0_ready_pct": 100.0 * resident[0] / max(1, desired[0]),
|
||||
"lod1_ready_pct": 100.0 * resident[1] / max(1, desired[1]),
|
||||
"lod2_ready_pct": 100.0 * resident[2] / max(1, desired[2]),
|
||||
"far_hole_pct": 100.0 * (far_desired - far_covered) / max(1, far_desired),
|
||||
})
|
||||
dwell_rows = [row for row in rows if row["phase"] == "dwell"]
|
||||
complete = next((row for row in dwell_rows
|
||||
if row["near_ready_pct"] >= 99.999
|
||||
and row["far_hole_pct"] <= 0.001), None)
|
||||
moving_work = [event for event in events
|
||||
if event.get("type") == "game_thread_work_frame"
|
||||
and move_start <= float(event.get("t_s", 0.0)) < stop]
|
||||
ticks = [float(event.get("tick_ms", 0.0)) for event in moving_work]
|
||||
updates = [float(event.get("update_chunks_ms", 0.0)) for event in moving_work]
|
||||
backend = next((event for event in events
|
||||
if event.get("type") == "worker_backend_config"), {})
|
||||
worker_cap_readback = next((event for event in events
|
||||
if event.get("type") == "streaming_worker_cap_readback"), {})
|
||||
if result.returncode or len(moving_samples) < 40 or len(dwell_rows) < 16:
|
||||
raise RuntimeError(f"Unreal run failed/incomplete: {trace}; exit={result.returncode}; "
|
||||
f"move={len(moving_samples)} dwell={len(dwell_rows)}")
|
||||
expected_split = (0 if condition == "baseline" else 1) \
|
||||
if ACTIVE_DESIGN == "priority_split" else 0
|
||||
expected_near_priority = "BackgroundLow" if expected_split == 0 else "BackgroundNormal"
|
||||
if (int(backend.get("streaming_tile_priority_split", -1)) != expected_split
|
||||
or backend.get("near_priority") != expected_near_priority
|
||||
or backend.get("horizon_priority") != "BackgroundLow"
|
||||
or int(worker_cap_readback.get("streaming_background_worker_cap", -1)) != worker_cap
|
||||
or int(backend.get("effective_tile_slots", -1)) != (8 if worker_cap else 10)):
|
||||
raise RuntimeError(f"priority CVar readback mismatch in {trace}: {backend}")
|
||||
return {
|
||||
"condition": condition, "platform": "unreal", "speed_mps": 30.0,
|
||||
"repetition": repetition, "core_limit": core_limit,
|
||||
"effective_tile_slots": backend.get("effective_tile_slots"),
|
||||
"streaming_background_worker_cap": worker_cap_readback.get(
|
||||
"streaming_background_worker_cap"),
|
||||
"taskgraph_background_threads": backend.get("taskgraph_background_threads"),
|
||||
"exit_code": result.returncode,
|
||||
"wall_s": round(time.monotonic() - started, 3),
|
||||
"game_thread_tick_p99_ms": p99(ticks),
|
||||
"game_thread_tick_max_ms": max(ticks, default=None),
|
||||
"stream_update_p99_ms": p99(updates),
|
||||
"stream_update_max_ms": max(updates, default=None),
|
||||
"dwell_complete_after_stop_s": complete.get("since_stop_s") if complete else None,
|
||||
"moving_samples": len(moving_samples), "dwell_samples": len(dwell_rows),
|
||||
"trace": str(trace.relative_to(ROOT)),
|
||||
"log": str(abslog.relative_to(ROOT)),
|
||||
**summarize(rows, "unreal"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
global ACTIVE_DESIGN, OUT, HOST_SAVED, HOST_OUT
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--platform", choices=("all", "headless", "unreal"), default="all")
|
||||
parser.add_argument("--design", choices=("priority_split", "worker_slots"),
|
||||
default="priority_split")
|
||||
args = parser.parse_args()
|
||||
ACTIVE_DESIGN = args.design
|
||||
design_path = "priority-split-verified" if ACTIVE_DESIGN == "priority_split" \
|
||||
else "worker-slots-verified"
|
||||
OUT = ROOT / "Saved/Linux/l120/movement" / design_path
|
||||
HOST_SAVED = HOST / "Saved/Linux/l120" / design_path
|
||||
HOST_OUT = HOST / "Plugins/VoxelForge/Saved/Linux/l120" / design_path
|
||||
if not DATA.is_file() or not HEADLESS.is_file() or not EDITOR.is_file():
|
||||
raise SystemExit("missing owner data, L108 headless binary, or Unreal editor")
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
results = []
|
||||
partial = OUT / "runs-partial.json"
|
||||
for repetition in range(1, 6):
|
||||
candidate = "priority_split" if ACTIVE_DESIGN == "priority_split" else "slots10"
|
||||
conditions = ("baseline", candidate) if repetition % 2 else (candidate, "baseline")
|
||||
platforms = (("headless", "unreal") if repetition % 2 else ("unreal", "headless")) \
|
||||
if args.platform == "all" else (args.platform,)
|
||||
for condition in conditions:
|
||||
for platform in platforms:
|
||||
row = (run_headless(condition, repetition) if platform == "headless"
|
||||
else run_unreal(condition, repetition))
|
||||
results.append(row)
|
||||
partial.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
|
||||
print("[L120Measurement] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
design_description = "near BGNormal vs horizon BackgroundLow" \
|
||||
if ACTIVE_DESIGN == "priority_split" else "effective UE tile slots 8 vs 10"
|
||||
(OUT / "summary.json").write_text(
|
||||
json.dumps({"design": design_description,
|
||||
"runs": results}, indent=2) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user