streaming: far LODs keep a share while near work is pending (L79); no more horizon starvation
L78's strict near-first order starved far LODs at 30 m/s (100% far holes, never completed). Unreal policy 2: once the collision ring is ready, LOD0-2 keep their reserve and far tiles are admitted as low-priority Horizon work; headless gives far work 1 of every 16 requests. Moving camera, far holes while moving: 1.4 m/s L77 8% / L78 29% / L79 31%; 30 m/s L77 8% / L78 100% / L79 72% (completes on dwell). Near readiness while moving at 30 m/s: 52 / 51 / 67%. Still worse than L77 for fast travel -> speed-adaptive policy next. World unchanged (v213). Fast tier PASS; Linux UE host builds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed
1 parent
ec20994a8e
commit
2c8a26c0d0
11 files changed
+833
-30
No files matched your search
@@ -208,6 +208,12 @@ namespace
|
||||
GVoxelForgeStreamingCoarseFirst,
|
||||
TEXT("Prioritize walkable tiles, then finish LOD0-2 before far clipmap work; controls far-level order."));
|
||||
|
||||
int32 GVoxelForgeStreamingSchedulerPolicy = 2;
|
||||
FAutoConsoleVariableRef CVarVoxelForgeStreamingSchedulerPolicy(
|
||||
TEXT("voxel.StreamingSchedulerPolicy"),
|
||||
GVoxelForgeStreamingSchedulerPolicy,
|
||||
TEXT("Streaming order: 0=L77 coarse-first, 1=L78 strict near-first, 2=L79 fair far share."));
|
||||
|
||||
int32 GVoxelForgeStreamingDynamicNearReserve = 1;
|
||||
FAutoConsoleVariableRef CVarVoxelForgeStreamingDynamicNearReserve(
|
||||
TEXT("voxel.StreamingDynamicNearReserve"),
|
||||
@@ -11949,6 +11955,9 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
|
||||
const int32 NearAdmissionCap = bPawnMoving ? MaxTasks : TotalAdmissionCap;
|
||||
const int32 CoarseTaskCap = FMath::Max(1, MaxTasks - NearTaskReserve) + ReadyQueueDepth;
|
||||
constexpr int32 FirstFarLevel = 3;
|
||||
const int32 SchedulerPolicy = FMath::Clamp(GVoxelForgeStreamingSchedulerPolicy, 0, 2);
|
||||
const int32 FirstCoarseLevel = FMath::Max(
|
||||
2, Settings ? Settings->FullResClipLevels : 2);
|
||||
const bool bUseCoarseFirstOrder = GVoxelForgeStreamingCoarseFirst != 0
|
||||
&& bWalkableRingReadyForReserve;
|
||||
int32 SchedulerSubmittedByLevel[9] = {};
|
||||
@@ -12449,34 +12458,60 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
|
||||
return true;
|
||||
};
|
||||
|
||||
// The support and collision-critical set drains first. Then LOD0, LOD1, and LOD2
|
||||
// use the near lane. Far tiles are not admitted until every requested near tile is ready.
|
||||
if (AreCriticalTilesReady() && bWalkableRingReadyForReserve)
|
||||
{
|
||||
for (int32 Level = 0; Level <= 2 && Level <= MaxDesiredLevel; ++Level)
|
||||
{
|
||||
const EVoxelTileWorkPriority Priority = Level == 0
|
||||
? EVoxelTileWorkPriority::LOD0 : EVoxelTileWorkPriority::Horizon;
|
||||
SubmitLevel(Level, Priority, NearAdmissionCap, true);
|
||||
}
|
||||
}
|
||||
|
||||
if (AreNearLODTilesReady())
|
||||
// Policy 0 reconstructs L77's coarse-first admission order for the moving-camera control.
|
||||
if (SchedulerPolicy == 0)
|
||||
{
|
||||
if (bUseCoarseFirstOrder)
|
||||
{
|
||||
for (int32 Level = MaxDesiredLevel; Level >= FirstFarLevel; --Level)
|
||||
for (int32 Level = MaxDesiredLevel; Level >= FirstCoarseLevel; --Level)
|
||||
{
|
||||
SubmitLevel(Level, EVoxelTileWorkPriority::Horizon,
|
||||
CoarseTaskCap, true);
|
||||
}
|
||||
for (int32 Level = 0; Level < FirstCoarseLevel && Level <= MaxDesiredLevel; ++Level)
|
||||
{
|
||||
const EVoxelTileWorkPriority Priority = Level == 0
|
||||
? EVoxelTileWorkPriority::LOD0 : EVoxelTileWorkPriority::Horizon;
|
||||
SubmitLevel(Level, Priority,
|
||||
Priority == EVoxelTileWorkPriority::LOD0 ? NearAdmissionCap : CoarseTaskCap,
|
||||
true);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
SubmitLevel(0, EVoxelTileWorkPriority::LOD0, NearAdmissionCap, false);
|
||||
for (int32 Level = 1; Level <= MaxDesiredLevel; ++Level)
|
||||
{
|
||||
SubmitLevel(Level, EVoxelTileWorkPriority::Horizon,
|
||||
CoarseTaskCap, false);
|
||||
if (GetGenerationAdmissionCount() >= CoarseTaskCap) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const bool bNearAdmissionReady = AreCriticalTilesReady()
|
||||
&& bWalkableRingReadyForReserve;
|
||||
if (bNearAdmissionReady)
|
||||
{
|
||||
for (int32 Level = 0; Level <= 2 && Level <= MaxDesiredLevel; ++Level)
|
||||
{
|
||||
const EVoxelTileWorkPriority Priority = Level == 0
|
||||
? EVoxelTileWorkPriority::LOD0 : EVoxelTileWorkPriority::Horizon;
|
||||
SubmitLevel(Level, Priority, NearAdmissionCap, true);
|
||||
}
|
||||
}
|
||||
|
||||
// L78 waits for all requested near tiles. L79 keeps a one-task far lane open
|
||||
// after critical requests; CoarseTaskCap reserves the remaining admissions for near work.
|
||||
const bool bFarAdmissionReady = SchedulerPolicy == 1
|
||||
? AreNearLODTilesReady() : true;
|
||||
if (bFarAdmissionReady)
|
||||
{
|
||||
for (int32 Level = FirstFarLevel; Level <= MaxDesiredLevel; ++Level)
|
||||
{
|
||||
SubmitLevel(Level, EVoxelTileWorkPriority::Horizon,
|
||||
CoarseTaskCap, false);
|
||||
CoarseTaskCap, bUseCoarseFirstOrder);
|
||||
if (GetGenerationAdmissionCount() >= CoarseTaskCap)
|
||||
{
|
||||
if (Level >= 0 && Level < 9) { ++SchedulerLevelCapBreaks[Level]; }
|
||||
@@ -12643,7 +12678,16 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
|
||||
> ResultBacklogCapStopsAtUpdateEntry;
|
||||
for (int32 Level = 0; Level < 9; ++Level)
|
||||
{
|
||||
AdmissionCapByLevel[Level] = Level <= 2 ? NearAdmissionCap : CoarseTaskCap;
|
||||
if (SchedulerPolicy == 0)
|
||||
{
|
||||
AdmissionCapByLevel[Level] = bUseCoarseFirstOrder
|
||||
? (Level >= FirstCoarseLevel || Level > 0 ? CoarseTaskCap : NearAdmissionCap)
|
||||
: (Level == 0 ? NearAdmissionCap : CoarseTaskCap);
|
||||
}
|
||||
else
|
||||
{
|
||||
AdmissionCapByLevel[Level] = Level <= 2 ? NearAdmissionCap : CoarseTaskCap;
|
||||
}
|
||||
if (DesiredMissingByLevel[Level] <= 0) continue;
|
||||
|
||||
if (bAllChunksLoaded)
|
||||
@@ -12730,7 +12774,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
|
||||
"\"core_region_concurrent_admission\":%d,"
|
||||
"\"max_tasks\":%d,\"ready_queue_depth\":%d,"
|
||||
"\"total_admission_cap\":%d,\"near_admission_cap\":%d,"
|
||||
"\"near_reserve\":%d,"
|
||||
"\"near_reserve\":%d,\"scheduler_policy\":%d,"
|
||||
"\"coarse_cap\":%d,\"first_far_level\":%d,"
|
||||
"\"submission_stop_level\":%d,\"submission_stop_cap\":%d,"
|
||||
"\"pending_entry\":%d,\"pending_after\":%d,\"admission_occupancy\":%d,"
|
||||
@@ -12754,7 +12798,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
|
||||
GVoxelForgeStreamingCoarseFirst, GVoxelForgeStreamingDynamicNearReserve,
|
||||
GVoxelForgeStreamingLegacyPendingResultCap,
|
||||
GVoxelForgeCoreRegionConcurrentAdmission, MaxTasks, ReadyQueueDepth,
|
||||
TotalAdmissionCap, NearAdmissionCap, NearTaskReserve, CoarseTaskCap, FirstFarLevel,
|
||||
TotalAdmissionCap, NearAdmissionCap, NearTaskReserve, SchedulerPolicy,
|
||||
CoarseTaskCap, FirstFarLevel,
|
||||
SchedulerOrderStopLevel, SchedulerOrderStopCap,
|
||||
PendingTilesAtUpdateEntry, PendingTiles.Num(), AdmissionOccupancy, ActiveTaskGuard,
|
||||
ActiveGenerationWorkerCount.load(std::memory_order_relaxed), PendingResultsEstimate,
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# Linux79 report
|
||||
|
||||
## Moving-camera starvation
|
||||
|
||||
The UE scheduler now defaults to policy 2: once the collision-critical ring is ready, LOD0–2 keep their reserved admission budget and far tiles are admitted as low-priority `Horizon` work. It no longer waits for every near tile before admitting far work. The headless policy uses a recurring far share (one far request per 16 near requests). Generation stays on workers.
|
||||
|
||||
The headless trace moved for 12 s, then dwelled for 5 s at 1.4 m/s and 30 m/s. Each run had 69 samples, 8 workers, and zero generation failures. Headless processes were pinned to CPUs 0–11, address-space limited to 6 GiB, and capped at 1,740 s. “Far holes” counts desired far tiles with neither their LOD nor a resident coarser parent. Near readiness is the moving-sample mean; dwell ended with near readiness at 100% in every run.
|
||||
|
||||
| Speed | Order | All far ready | Moving far holes, mean | Max far stale age | Moving near ready, mean | Dwell far holes |
|
||||
|---|---|---:|---:|---:|---:|---:|
|
||||
| 1.4 m/s | L77 coarse-first | 1.50 s | 8.25% | 1.25 s | 73.73% | 0% |
|
||||
| 1.4 m/s | L78 strict near-first | 4.00 s | 29.20% | 3.75 s | 85.03% | 0% |
|
||||
| 1.4 m/s | L79 fair share | 4.25 s | 30.79% | 4.00 s | 82.90% | 0% |
|
||||
| 30 m/s | L77 coarse-first | 1.75 s | 8.16% | 1.25 s | 52.14% | 0% |
|
||||
| 30 m/s | L78 strict near-first | Never | 100.00% | 17.00 s | 51.40% | 100% |
|
||||
| 30 m/s | L79 fair share | 9.75 s | 71.89% | 9.50 s | 66.67% | 0% |
|
||||
|
||||
Verdict: L78 starves the horizon at 30 m/s, including through the dwell. L79 prevents indefinite starvation and finishes far coverage while still moving, but far coverage remains sparse during fast travel. L77 has much better far coverage and faster far completion, while L79 improves near readiness at 30 m/s. At walking speed L79 is close to L78. L79 is the near-priority/far-progress compromise; it does not provide L77-level moving coverage.
|
||||
|
||||
Trace summary and samples: `Saved/Linux/l79/movement/headless/summary.json` and `samples.csv`; per-policy logs are alongside them.
|
||||
|
||||
## Passage-cost decision
|
||||
|
||||
I tested a point-bound filter candidate against the same page route on 32 tiles (8 per strate: four LOD0 and four LOD1). Across 42,875 density samples per tile it changed zero samples and zero signs. Maximum and mean surface displacement were both 0 cm; solid/air membership did not change, and there were no changed samples at mouths or within 4 m of walls. There were consequently no changed tiles to classify as mouth or wall losses.
|
||||
|
||||
The candidate did not reduce passage work: candidate/base median tile-time ratios were SW 1.002, FI 1.014, TN 0.990, and VS 0.993. All inspected pairs also reported zero baseline and candidate segment visits, so this sample set is a negative control and does not validate bounds on an active passage-wall sample. I rejected the candidate: it showed no useful performance gain, and it is not accepted as proof that the larger passage cuts are safe. No field change or evaluator-version bump was made; the passage cost remains open.
|
||||
|
||||
The broad L76 cut remains rejected (612 density mismatches; mesh, payload, and topology changed on 24 tiles). Its retained artifacts contain aggregate hash counts, not per-sample deltas, so this round does not claim to have inspected those individual changes. The L79 candidate’s per-tile evidence is in `Saved/Linux/l79/passage-bounds/{0-SurfaceWorld,1-FloatingIslands,2-TunnelNetwork,3-VerticalShafts}.log`.
|
||||
|
||||
No new mouth or mid-tunnel capture set was made because no passage geometry change was accepted. The final fast tier’s existing passage audits passed, including support, capsule/walkability, grade/floor, ripple, naturalness, route binding, and mouth-profile gates.
|
||||
|
||||
## Readiness, W8, three interleaved pairs, median seconds
|
||||
|
||||
| Strate | L78 LOD0–2 | L79 LOD0–2 | L78 whole view | L79 whole view |
|
||||
|---|---:|---:|---:|---:|
|
||||
| SurfaceWorld | 1.232 | 1.365 | 2.762 | 2.611 |
|
||||
| FloatingIslands | 3.391 | 3.503 | 4.881 | 4.763 |
|
||||
| TunnelNetwork | 3.363 | 3.737 | 4.621 | 4.792 |
|
||||
| VerticalShafts | 1.417 | 1.683 | 2.615 | 2.603 |
|
||||
|
||||
L79 trades a small near-readiness regression for guaranteed far admission. The L77 sampler census from the L78 before/after pair set is retained here; B/A are before/after the exact composer trims. “Near” means LOD0–2.
|
||||
|
||||
| Strate | Near grid-fill closure B/A | Whole closure B/A | Passage CPU share B/A | Support CPU share B/A |
|
||||
|---|---:|---:|---:|---:|
|
||||
| SurfaceWorld | 99.1% / 99.9% | 100.1% / 99.6% | 45.1% / 47.6% | 15.0% / 12.8% |
|
||||
| FloatingIslands | 99.9% / 99.5% | 99.8% / 99.8% | 64.6% / 63.0% | 14.1% / 13.8% |
|
||||
| TunnelNetwork | 99.9% / 100.1% | 99.8% / 100.2% | 65.9% / 63.7% | 13.2% / 13.5% |
|
||||
| VerticalShafts | 100.0% / 100.2% | 99.9% / 100.9% | 45.9% / 42.9% | 5.9% / 6.6% |
|
||||
|
||||
Timing data: `Saved/Linux/l79/timing/summary.json`. Sampler source data: `Saved/Linux/l78/{sampler-census,closure-census,route-census}.csv`; sampler handler cost stayed below 0.05% of profiled thread CPU.
|
||||
|
||||
## Version, hashes, and verification
|
||||
|
||||
- `WorldAssemblyEvaluatorVersion=213`; unchanged. The passage candidate was removed, so no geometry or world-hash update was accepted.
|
||||
- `Standalone/data/owner_world.json` SHA256: `38b3d3085f2a0711a7c8dbe8883aaaf2510a9acbfb529ebece6de6d8c2dbfde`.
|
||||
- Final source stamp: `084e7beed3039198b633d0075793f14796fdd7628b307406f6fdda7aedab641a`.
|
||||
- Fast tier: PASS, 274 checks, 332.3 s. Passage audits and Clang W1/W8/W12 determinism checks passed. Clang and GCC binaries built at the final source stamp; no L79 GCC W1/W8/W12 matrix or new passage captures were run because no geometry change was accepted. Clang headless SHA256 `f67c58b3b60bf41bcdb630f82afee5b87a57c3b991dad3dbb4e93523032c9229`; GCC `faf139e63a7eff5c27f898bef508240ae602b63d9e509bc512dec71118fe4fe3`.
|
||||
- Linux `VFHostEditor` build after timing: PASS in 81.49 s, `taskset -c 0-11`, `-MaxParallelActions=4`, no address-space limit. Three existing RealtimeMesh deprecation warnings. The only changed `Source/` file is `Source/VoxelForge/Private/VoxelWorld.cpp`; UBT compiled it and the VFCore amalgam. Log: `Saved/Linux/l79/build/ue-host-build-final.log`.
|
||||
|
||||
## Failed runs
|
||||
|
||||
- Three Unreal null-RHI moving-trace attempts exited during asset load at the 6 GiB address-space cap (`Ran out of memory allocating 5,096,960 bytes`). The permitted headless moving-camera traces supplied the comparison instead.
|
||||
- The first headless movement diagnostic segfaulted after generator release because it updated desired tiles without the camera overload. The runner was corrected to pass the camera; all six final traces passed.
|
||||
- The first fast-tier launch rejected a stale expected binary stamp. Clang/GCC binaries were rebuilt; the final 274-check run passed.
|
||||
- The broad route-parity diagnostic also failed on both the L79 baseline and unmodified L78 reference: FI had 3/8 base/page mismatches and 5,500 exact-query mismatches. This is pre-existing cross-route diagnostic behavior; the same-route point-bound inspection completed, and the final fast tier passed.
|
||||
|
||||
No commit, push, or stash was made. The branch remains `assembler/world` at `ec20994`.
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure streaming coverage while the standalone queue follows a moving camera."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l79/movement/headless"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BINARY = ROOT / "Saved/Linux/l79/bin/VoxelForgeCoreHeadless-l79-final"
|
||||
CAMERA_M = (19.166015625, -14.2001953125, -753.3740234375)
|
||||
POLICIES = {0: "L77", 1: "L78", 2: "L79"}
|
||||
SPEEDS = {"walk": 1.4, "fast": 30.0}
|
||||
SAMPLE = re.compile(r"^\[L79MovingSample\] (.*)$", re.M)
|
||||
SUMMARY = re.compile(r"^\[L79MovingSummary\] (.*)$", re.M)
|
||||
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
|
||||
|
||||
def run(policy: int, speed_name: str) -> tuple[dict, list[dict]]:
|
||||
label = f"{speed_name}-{POLICIES[policy]}"
|
||||
log = OUT / f"{label}.log"
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
camera = ",".join(f"{value:.12f}" for value in CAMERA_M) + ",0,0"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||
"timeout", "1740s", str(BINARY), str(DATA),
|
||||
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
|
||||
"--workers=8", "--clip-radius=3", "--max-clip-level=8", "--max-level=8",
|
||||
f"--camera={camera}", "--profile-lod-work",
|
||||
f"--stream-scheduler-policy={policy}",
|
||||
f"--moving-camera-speed-mps={SPEEDS[speed_name]:.1f}",
|
||||
"--moving-camera-duration-s=12", "--moving-camera-dwell-s=5",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as stream:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=stream,
|
||||
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||
content = log.read_text(encoding="utf-8", errors="replace")
|
||||
samples = [dict(FIELDS.findall(match.group(1))) for match in SAMPLE.finditer(content)]
|
||||
summary_match = SUMMARY.search(content)
|
||||
engine_summary = dict(FIELDS.findall(summary_match.group(1))) if summary_match else {}
|
||||
if result.returncode != 0 or not samples or not engine_summary:
|
||||
raise SystemExit(f"moving-camera run failed or trace missing: {log}; exit={result.returncode}")
|
||||
for row in samples:
|
||||
for name in ("policy", "speed_mps", "t_s", "x_m", "near_ready_pct", "far_missing_pct",
|
||||
"far_hole_pct",
|
||||
"far_stale_mean_s", "far_stale_max_s", "generation_failures", "queue_update_ms"):
|
||||
row[name] = float(row[name])
|
||||
row["label"] = label
|
||||
if int(float(engine_summary.get("generation_failures", "-1"))) != 0:
|
||||
raise SystemExit(f"tile generation failure: {log}")
|
||||
run_info = {
|
||||
"label": label, "policy": POLICIES[policy], "policy_id": policy,
|
||||
"speed": speed_name, "speed_mps": SPEEDS[speed_name],
|
||||
"exit_code": result.returncode, "elapsed_s": round(time.monotonic() - started, 3),
|
||||
"samples": len(samples), "engine_summary": engine_summary,
|
||||
"log": str(log.relative_to(ROOT)),
|
||||
}
|
||||
print("[L79HeadlessMoveRun] " + json.dumps(run_info, sort_keys=True), flush=True)
|
||||
return run_info, samples
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not DATA.is_file() or not BINARY.is_file():
|
||||
raise SystemExit(f"missing input: {DATA} or {BINARY}")
|
||||
results: list[dict] = []
|
||||
samples: list[dict] = []
|
||||
schedule = [(policy, speed) for speed in ("walk", "fast")
|
||||
for policy in (0, 1, 2)]
|
||||
for policy, speed in schedule:
|
||||
result, rows = run(policy, speed)
|
||||
results.append(result)
|
||||
samples.extend(rows)
|
||||
(OUT / "runs-partial.json").write_text(
|
||||
json.dumps(results, indent=2) + "\n", encoding="utf-8")
|
||||
summary = []
|
||||
for speed_name, speed in SPEEDS.items():
|
||||
for policy in POLICIES:
|
||||
rows = [row for row in samples
|
||||
if row["speed_mps"] == speed and row["policy"] == policy]
|
||||
moving = [row for row in rows if row.get("phase") == "move"]
|
||||
dwell = [row for row in rows if row.get("phase") == "dwell"]
|
||||
summary.append({
|
||||
"speed": speed_name, "speed_mps": speed, "policy": POLICIES[policy],
|
||||
"moving_samples": len(moving), "dwell_samples": len(dwell),
|
||||
"far_missing_moving_mean_pct": round(
|
||||
sum(row["far_missing_pct"] for row in moving) / max(1, len(moving)), 3),
|
||||
"far_missing_moving_max_pct": max(
|
||||
(row["far_missing_pct"] for row in moving), default=None),
|
||||
"far_missing_dwell_last_pct": dwell[-1]["far_missing_pct"] if dwell else None,
|
||||
"far_hole_moving_mean_pct": round(
|
||||
sum(row["far_hole_pct"] for row in moving) / max(1, len(moving)), 3),
|
||||
"far_hole_moving_max_pct": max(
|
||||
(row["far_hole_pct"] for row in moving), default=None),
|
||||
"far_hole_dwell_last_pct": dwell[-1]["far_hole_pct"] if dwell else None,
|
||||
"far_stale_moving_max_s": max(
|
||||
(row["far_stale_max_s"] for row in moving), default=None),
|
||||
"far_stale_dwell_max_s": max(
|
||||
(row["far_stale_max_s"] for row in dwell), default=None),
|
||||
"near_ready_moving_mean_pct": round(
|
||||
sum(row["near_ready_pct"] for row in moving) / max(1, len(moving)), 3),
|
||||
"near_ready_moving_min_pct": min(
|
||||
(row["near_ready_pct"] for row in moving), default=None),
|
||||
"near_ready_dwell_last_pct": dwell[-1]["near_ready_pct"] if dwell else None,
|
||||
"queue_update_max_ms": max((row["queue_update_ms"] for row in rows), default=None),
|
||||
})
|
||||
(OUT / "summary.json").write_text(
|
||||
json.dumps({"runs": results, "summary": summary}, indent=2) + "\n", encoding="utf-8")
|
||||
if samples:
|
||||
with (OUT / "samples.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(samples[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(samples)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare L77, L78, and L79 production streaming orders while the pawn moves."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
HOST = Path("/home/jahni/VFHost")
|
||||
PLUGIN_SAVED = ROOT / "Saved/Linux/l79"
|
||||
HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l79"
|
||||
HOST_SAVED = HOST / "Saved/Linux/l79"
|
||||
EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd")
|
||||
SPAWN_M = (19.166015625, -14.2001953125, -753.3740234375) # owner TunnelNetwork
|
||||
SPEEDS = {"walk": 140.0, "fast": 3000.0} # cm/s: 1.4 m/s and 30 m/s
|
||||
POLICY_NAMES = {0: "L77", 1: "L78", 2: "L79"}
|
||||
TRACE_DIR = PLUGIN_SAVED / "movement/traces"
|
||||
MOVE_START_S = 6.0
|
||||
EXIT_S = 18.0
|
||||
|
||||
|
||||
def parse_trace(path: Path, run: dict) -> tuple[list[dict], dict]:
|
||||
events = []
|
||||
if path.is_file():
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
try:
|
||||
events.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
samples = [event for event in events if event.get("type") == "streaming_sample"
|
||||
and float(event.get("speed_cm_s", 0.0)) >= 50.0]
|
||||
scheduler = [event for event in events if event.get("type") == "scheduler_admission"
|
||||
and float(event.get("moving", 0)) > 0]
|
||||
rows = []
|
||||
for event in samples:
|
||||
desired = event.get("desired_by_lod", [0] * 9)
|
||||
resident = event.get("resident_by_lod", [0] * 9)
|
||||
pending = event.get("pending_by_lod", [0] * 9)
|
||||
far_desired = sum(desired[3:])
|
||||
far_resident = sum(resident[3:])
|
||||
near_desired = sum(desired[:3])
|
||||
near_resident = sum(resident[:3])
|
||||
rows.append({
|
||||
"run": run["label"], "policy": run["policy_name"], "speed": run["speed"],
|
||||
"t_s": event.get("t_s"), "position_cm": event.get("position_cm"),
|
||||
"speed_cm_s": event.get("speed_cm_s"),
|
||||
"far_desired": far_desired, "far_resident": far_resident,
|
||||
"far_pending": sum(pending[3:]),
|
||||
"far_missing_pct": round(100.0 * (far_desired - far_resident)
|
||||
/ max(1, far_desired), 3),
|
||||
"near_desired": near_desired, "near_resident": near_resident,
|
||||
"near_ready_pct": round(100.0 * near_resident / max(1, near_desired), 3),
|
||||
"walkable_ready": event.get("walkable_ready"),
|
||||
**{f"lod{i}_desired": desired[i] for i in range(9)},
|
||||
**{f"lod{i}_resident": resident[i] for i in range(9)},
|
||||
})
|
||||
submitted_far = sum(sum(event.get("submitted_by_lod", [0] * 9)[3:])
|
||||
for event in scheduler)
|
||||
submitted_near = sum(sum(event.get("submitted_by_lod", [0] * 9)[:3])
|
||||
for event in scheduler)
|
||||
far_missing = [row["far_missing_pct"] for row in rows]
|
||||
near_ready = [row["near_ready_pct"] for row in rows]
|
||||
abslog_lines = (run["abslog_path"].read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
if run["abslog_path"].is_file() else [])
|
||||
summary = {
|
||||
"moving_samples": len(rows),
|
||||
"moving_duration_s": round(max((float(row["t_s"]) for row in rows), default=0.0)
|
||||
- min((float(row["t_s"]) for row in rows), default=0.0), 3),
|
||||
"far_missing_pct_mean": round(sum(far_missing) / max(1, len(far_missing)), 3),
|
||||
"far_missing_pct_max": max(far_missing, default=None),
|
||||
"far_missing_pct_first": far_missing[0] if far_missing else None,
|
||||
"far_missing_pct_last": far_missing[-1] if far_missing else None,
|
||||
"near_ready_pct_mean": round(sum(near_ready) / max(1, len(near_ready)), 3),
|
||||
"near_ready_pct_min": min(near_ready, default=None),
|
||||
"near_ready_pct_last": near_ready[-1] if near_ready else None,
|
||||
"walkable_ready_samples": sum(row["walkable_ready"] == 1 for row in rows),
|
||||
"submitted_near_tiles": submitted_near,
|
||||
"submitted_far_tiles": submitted_far,
|
||||
"lod0_ring_ready_events": sum(event.get("type") == "lod0_ring_ready" for event in events),
|
||||
"visual_gap_result": next((line.split("result=")[1].split()[0]
|
||||
for line in abslog_lines
|
||||
if "[VoxelForgeVisualGapAudit] result=" in line), "not-run"),
|
||||
"visual_staleness_result": next((line.split("result=")[1].split()[0]
|
||||
for line in abslog_lines
|
||||
if "[VoxelForgeVisualStaleness] result=" in line), "not-run"),
|
||||
}
|
||||
return rows, summary
|
||||
|
||||
|
||||
def run(policy: int, speed_name: str, replicate: int, audit: bool = False) -> tuple[dict, list[dict]]:
|
||||
label = f"{speed_name}-{POLICY_NAMES[policy]}-r{replicate}" + ("-audit" if audit else "")
|
||||
trace = TRACE_DIR / f"{label}.trace.jsonl"
|
||||
abslog = TRACE_DIR / f"{label}.log"
|
||||
stdout = TRACE_DIR / f"{label}.stdout.log"
|
||||
output = HOST_PLUGIN_SAVED / "output" / label
|
||||
userdir = HOST_SAVED / "user" / label
|
||||
zendata = HOST_SAVED / "zen" / label
|
||||
ddc = HOST_SAVED / "DDC"
|
||||
for directory in (TRACE_DIR, output, userdir, zendata, ddc):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M)
|
||||
command = [
|
||||
str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended",
|
||||
"-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch",
|
||||
"-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer",
|
||||
"-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}",
|
||||
f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0",
|
||||
"-archetype=TunnelNetwork", "-slot=0", "-modes=export", "-gameconfig=1",
|
||||
"-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25",
|
||||
"-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0",
|
||||
"-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0",
|
||||
f"-out={output}", "-voxel.StreamingCoarseFirst=1",
|
||||
"-voxel.StreamingDynamicNearReserve=1", "-voxel.StreamingLegacyPendingResultCap=0",
|
||||
"-voxel.CoreRegionConcurrentAdmission=1", "-voxel.StreamingResultBacklogCap=256",
|
||||
"-voxel.StreamingReadyQueueDepth=8", "-voxel.StreamingBackgroundWorkerCap=1",
|
||||
"-voxel.CoreColumnCacheDirectMapped=0", "-voxel.CoreMCGradientMemoization=1",
|
||||
"-voxel.StreamingCollisionLookahead=1", "-voxel.TestMove=1",
|
||||
f"-voxel.TestMoveSpeedCmPerSecond={SPEEDS[speed_name]:.1f}",
|
||||
f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}",
|
||||
f"-voxel.TestExitSeconds={EXIT_S:.1f}", "-voxel.TestMoveDirection=1,0,0",
|
||||
f"-voxel.StreamingSchedulerPolicy={policy}", "-voxel.TestExitOnSteady=0",
|
||||
"-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}",
|
||||
"-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1",
|
||||
"-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}",
|
||||
f"-abslog={abslog}",
|
||||
]
|
||||
if audit:
|
||||
command.extend(("-voxel.TestVisualStaleness=2", "-voxel.TestVisualGapAudit=1"))
|
||||
wrapped = ["taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||
"timeout", "1740s", *command]
|
||||
started = time.monotonic()
|
||||
with stdout.open("w", encoding="utf-8") as stream:
|
||||
result = subprocess.run(wrapped, cwd=ROOT, stdout=stream,
|
||||
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||
run_info = {
|
||||
"label": label, "policy": policy, "policy_name": POLICY_NAMES[policy],
|
||||
"speed": speed_name, "speed_cm_s": SPEEDS[speed_name], "replicate": replicate,
|
||||
"audit": audit, "exit_code": result.returncode,
|
||||
"elapsed_s": round(time.monotonic() - started, 3),
|
||||
"trace": str(trace.relative_to(ROOT)), "abslog": str(abslog.relative_to(ROOT)),
|
||||
"stdout": str(stdout.relative_to(ROOT)), "abslog_path": abslog,
|
||||
}
|
||||
samples, summary = parse_trace(trace, run_info)
|
||||
run_info.update(summary)
|
||||
del run_info["abslog_path"]
|
||||
print("[L79MoveRun] " + json.dumps(run_info, sort_keys=True), flush=True)
|
||||
return run_info, samples
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not EDITOR.is_file():
|
||||
raise SystemExit(f"missing Unreal commandlet executable: {EDITOR}")
|
||||
TRACE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
results: list[dict] = []
|
||||
sample_rows: list[dict] = []
|
||||
schedule = []
|
||||
for replicate, policies in ((1, (0, 1, 2)), (2, (2, 1, 0))):
|
||||
speeds = ("walk", "fast") if replicate == 1 else ("fast", "walk")
|
||||
for policy in policies:
|
||||
for speed in speeds:
|
||||
schedule.append((policy, speed, replicate, False))
|
||||
schedule.append((2, "fast", 3, True))
|
||||
for policy, speed, replicate, audit in schedule:
|
||||
row, samples = run(policy, speed, replicate, audit)
|
||||
results.append(row)
|
||||
sample_rows.extend(samples)
|
||||
(PLUGIN_SAVED / "movement/runs-partial.json").write_text(
|
||||
json.dumps(results, indent=2) + "\n", encoding="utf-8")
|
||||
(PLUGIN_SAVED / "movement/summary.json").write_text(
|
||||
json.dumps({"runs": results}, indent=2) + "\n", encoding="utf-8")
|
||||
sample_path = PLUGIN_SAVED / "movement/samples.csv"
|
||||
if sample_rows:
|
||||
with sample_path.open("w", newline="", encoding="utf-8") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(sample_rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(sample_rows)
|
||||
return 0 if all(row["exit_code"] == 0 and row["moving_samples"] > 0
|
||||
for row in results) else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Interleave L78/L79 owner-world readiness timings for all four strates."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
OUT = ROOT / "Saved/Linux/l79/timing"
|
||||
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||
BINARIES = {
|
||||
"L78": ROOT / "Saved/Linux/l78/bin/VoxelForgeCoreHeadless-l78-clang-profile",
|
||||
"L79": ROOT / "Saved/Linux/l79/bin/VoxelForgeCoreHeadless-l79-clang-profile",
|
||||
}
|
||||
CAMERAS = {
|
||||
0: (8.0, 16.0, -372.2626953125),
|
||||
1: (25.8369140625, 3.19921875, -737.0),
|
||||
2: (19.166015625, -14.2001953125, -753.3740234375),
|
||||
3: (-20.0625, -5.068359375, -900.353515625),
|
||||
}
|
||||
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||
LOD_READY = re.compile(
|
||||
r"^\[WorldAssemblerL77LODReady\] lod=(\d+) tiles=(\d+) cold_ready_s=([0-9.]+)$",
|
||||
re.M,
|
||||
)
|
||||
NEAR_READY = re.compile(
|
||||
r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$",
|
||||
re.M,
|
||||
)
|
||||
ORDERS = (("L78", "L79"), ("L79", "L78"), ("L78", "L79"))
|
||||
|
||||
|
||||
def run(strate: int, pair: int, mode: str) -> dict:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
log = OUT / f"strate{strate}-pair{pair}-{mode}.log"
|
||||
camera = ",".join(f"{value:.12f}" for value in CAMERAS[strate]) + ",0,0"
|
||||
command = [
|
||||
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||
"timeout", "1740s", str(BINARIES[mode]), str(DATA),
|
||||
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
|
||||
"--workers=8", "--clip-radius=3", "--max-clip-level=8", "--max-level=8",
|
||||
f"--camera={camera}", "--profile-lod-work", "--linux77-profile",
|
||||
]
|
||||
started = time.monotonic()
|
||||
with log.open("w", encoding="utf-8") as output:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=output,
|
||||
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||
content = log.read_text(encoding="utf-8", errors="replace")
|
||||
pipeline_match = PIPELINE.search(content)
|
||||
ready = {int(lod): {"tiles": int(tiles), "seconds": float(seconds)}
|
||||
for lod, tiles, seconds in LOD_READY.findall(content)}
|
||||
near_match = NEAR_READY.search(content)
|
||||
if (result.returncode != 0 or pipeline_match is None or near_match is None
|
||||
or any(lod not in ready for lod in (0, 1, 2))):
|
||||
raise SystemExit(f"readiness run failed or data missing: {log}; exit={result.returncode}")
|
||||
pipeline = dict(FIELDS.findall(pipeline_match.group(1)))
|
||||
row = {
|
||||
"strate": strate,
|
||||
"pair": pair,
|
||||
"mode": mode,
|
||||
"exit_code": result.returncode,
|
||||
"wall_s": round(time.monotonic() - started, 3),
|
||||
"lod02_ready_s": float(near_match.group(3)),
|
||||
"lod_ready_s": {str(lod): ready[lod]["seconds"] for lod in sorted(ready)},
|
||||
"whole_view_ready_s": float(pipeline["whole_view_ready"].removesuffix("s")),
|
||||
"whole_tiles": int(pipeline["whole_tiles"].split("/")[0]),
|
||||
"sampler": next((line for line in content.splitlines()
|
||||
if line.startswith("[L77Sampler] action=stop")), None),
|
||||
"log": str(log.relative_to(ROOT)),
|
||||
}
|
||||
print("[L79TimingRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def main() -> int:
|
||||
missing = [str(path) for path in (DATA, *BINARIES.values()) if not path.is_file()]
|
||||
if missing:
|
||||
raise SystemExit("missing input: " + ", ".join(missing))
|
||||
rows = []
|
||||
for strate in range(4):
|
||||
for pair, order in enumerate(ORDERS, 1):
|
||||
paired = {}
|
||||
for mode in order:
|
||||
row = run(strate, pair, mode)
|
||||
rows.append(row)
|
||||
paired[mode] = row
|
||||
(OUT / "runs-partial.json").write_text(
|
||||
json.dumps(rows, indent=2) + "\n", encoding="utf-8")
|
||||
print("[L79TimingPair] " + json.dumps({
|
||||
"strate": strate, "pair": pair,
|
||||
"L78_lod02_s": paired["L78"]["lod02_ready_s"],
|
||||
"L79_lod02_s": paired["L79"]["lod02_ready_s"],
|
||||
"L78_whole_s": paired["L78"]["whole_view_ready_s"],
|
||||
"L79_whole_s": paired["L79"]["whole_view_ready_s"],
|
||||
}, sort_keys=True), flush=True)
|
||||
summary = []
|
||||
for strate in range(4):
|
||||
group = [row for row in rows if row["strate"] == strate]
|
||||
summary.append({
|
||||
"strate": strate,
|
||||
"pairs": 3,
|
||||
"L78_lod02_s": [row["lod02_ready_s"] for row in group if row["mode"] == "L78"],
|
||||
"L79_lod02_s": [row["lod02_ready_s"] for row in group if row["mode"] == "L79"],
|
||||
"L78_lod02_median_s": statistics.median(
|
||||
row["lod02_ready_s"] for row in group if row["mode"] == "L78"),
|
||||
"L79_lod02_median_s": statistics.median(
|
||||
row["lod02_ready_s"] for row in group if row["mode"] == "L79"),
|
||||
"L78_whole_s": [row["whole_view_ready_s"] for row in group if row["mode"] == "L78"],
|
||||
"L79_whole_s": [row["whole_view_ready_s"] for row in group if row["mode"] == "L79"],
|
||||
"L78_whole_median_s": statistics.median(
|
||||
row["whole_view_ready_s"] for row in group if row["mode"] == "L78"),
|
||||
"L79_whole_median_s": statistics.median(
|
||||
row["whole_view_ready_s"] for row in group if row["mode"] == "L79"),
|
||||
})
|
||||
(OUT / "summary.json").write_text(
|
||||
json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -86,6 +86,11 @@ namespace
|
||||
bool HasCameraStep=false;
|
||||
bool AutoSurfaceCamera=false;
|
||||
bool CaptureSkipWalkability=false;
|
||||
int StreamingSchedulerPolicy=-1;
|
||||
bool MovingCameraTrace=false;
|
||||
double MovingCameraSpeedMetersPerSecond=0.0;
|
||||
double MovingCameraDurationSeconds=12.0;
|
||||
double MovingCameraDwellSeconds=5.0;
|
||||
FVector CameraMeters=FVector::ZeroVector;
|
||||
FVector CameraStepMeters=FVector::ZeroVector;
|
||||
float CameraYawDegrees=0.0f;
|
||||
@@ -1409,6 +1414,8 @@ namespace
|
||||
std::printf("Usage: %s [owner_world.json] --world-assembler-prototype --stream-benchmark --stream-pipeline [options]\n"
|
||||
" --workers=N --clip-radius=N --max-clip-level=N --max-level=N\n"
|
||||
" --camera=X,Y,Z,YAW,PITCH (position in metres) --stream-hash\n"
|
||||
" --stream-scheduler-policy=0|1|2 (L77 coarse, L78 near, L79 fair share)\n"
|
||||
" --moving-camera-speed-mps=N [--moving-camera-duration-s=N] [--moving-camera-dwell-s=N]\n"
|
||||
" --stream-hash-tiles (print per-tile hashes; implies --stream-hash)\n"
|
||||
" --stream-hash-tiles-allow-skips (mesh/topology falsifier with proven uniform skips)\n"
|
||||
" --camera-step=X,Y,Z (optional second view step in metres)\n"
|
||||
@@ -1590,6 +1597,37 @@ namespace
|
||||
{std::fprintf(stderr,"Invalid --camera-step value: %.*s\n",static_cast<int>(arg.size()),arg.data());return false;}
|
||||
options.HasCameraStep=true;
|
||||
}
|
||||
else if(arg.starts_with("--stream-scheduler-policy="))
|
||||
{
|
||||
if(!readInt("--stream-scheduler-policy=",options.StreamingSchedulerPolicy)
|
||||
||options.StreamingSchedulerPolicy<0||options.StreamingSchedulerPolicy>2)
|
||||
{std::fprintf(stderr,"Invalid --stream-scheduler-policy (expected 0..2).\n");return false;}
|
||||
}
|
||||
else if(arg.starts_with("--moving-camera-speed-mps="))
|
||||
{
|
||||
const std::string value(arg.substr(std::string_view("--moving-camera-speed-mps=").size()));
|
||||
char* end=nullptr;options.MovingCameraSpeedMetersPerSecond=std::strtod(value.c_str(),&end);
|
||||
if(end==value.c_str()||*end!='\0'||!std::isfinite(options.MovingCameraSpeedMetersPerSecond)
|
||||
||options.MovingCameraSpeedMetersPerSecond<=0.0)
|
||||
{std::fprintf(stderr,"Invalid --moving-camera-speed-mps (expected positive finite value).\n");return false;}
|
||||
options.MovingCameraTrace=true;
|
||||
}
|
||||
else if(arg.starts_with("--moving-camera-duration-s="))
|
||||
{
|
||||
const std::string value(arg.substr(std::string_view("--moving-camera-duration-s=").size()));
|
||||
char* end=nullptr;options.MovingCameraDurationSeconds=std::strtod(value.c_str(),&end);
|
||||
if(end==value.c_str()||*end!='\0'||!std::isfinite(options.MovingCameraDurationSeconds)
|
||||
||options.MovingCameraDurationSeconds<=0.0)
|
||||
{std::fprintf(stderr,"Invalid --moving-camera-duration-s (expected positive finite value).\n");return false;}
|
||||
}
|
||||
else if(arg.starts_with("--moving-camera-dwell-s="))
|
||||
{
|
||||
const std::string value(arg.substr(std::string_view("--moving-camera-dwell-s=").size()));
|
||||
char* end=nullptr;options.MovingCameraDwellSeconds=std::strtod(value.c_str(),&end);
|
||||
if(end==value.c_str()||*end!='\0'||!std::isfinite(options.MovingCameraDwellSeconds)
|
||||
||options.MovingCameraDwellSeconds<0.0)
|
||||
{std::fprintf(stderr,"Invalid --moving-camera-dwell-s (expected nonnegative finite value).\n");return false;}
|
||||
}
|
||||
else if(arg.starts_with("--column-cache="))
|
||||
{
|
||||
const std::string_view mode=arg.substr(std::string_view("--column-cache=").size());
|
||||
@@ -1635,6 +1673,10 @@ namespace
|
||||
{std::fprintf(stderr,"--tls-probe-touch requires --tls-probe=N\n");return false;}
|
||||
if(options.RegionCrossing&&!options.HasCameraStep)
|
||||
{std::fprintf(stderr,"--region-crossing requires --camera-step=X,Y,Z\n");return false;}
|
||||
if(options.MovingCameraTrace&&(!options.StreamPipeline||!options.WorldAssemblerPrototype))
|
||||
{std::fprintf(stderr,"--moving-camera-speed-mps requires --world-assembler-prototype --stream-pipeline\n");return false;}
|
||||
if(options.MovingCameraTrace&&options.StreamingSchedulerPolicy<0)
|
||||
{std::fprintf(stderr,"--moving-camera-speed-mps requires --stream-scheduler-policy=0|1|2\n");return false;}
|
||||
if(options.LandmarkL45RampControl&&!options.LandmarkChamberDemo)
|
||||
{std::fprintf(stderr,"--landmark-l45-ramp-control requires --landmark-chamber-demo\n");return false;}
|
||||
if(options.AuditOreVeinYield&&!options.OreVeinsDemo)
|
||||
@@ -2228,5 +2270,8 @@ int main(int argc,char** argv)
|
||||
?options.ParallelSignEdgeRepair:options.FloatingIslandsRecipe,
|
||||
options.AuditNativeLattice,options.LatticeOrphanControl,
|
||||
options.CaptureSkipWalkability,options.ProfilePassageComposer,
|
||||
options.StreamHashAllowUniformSkips,options.Linux77Profile);
|
||||
options.StreamHashAllowUniformSkips,options.Linux77Profile,
|
||||
options.StreamingSchedulerPolicy,options.MovingCameraTrace,
|
||||
options.MovingCameraSpeedMetersPerSecond,options.MovingCameraDurationSeconds,
|
||||
options.MovingCameraDwellSeconds);
|
||||
}
|
||||
@@ -1874,7 +1874,9 @@ namespace StandaloneStreamBenchmark
|
||||
bool bParallelSignEdgeRepair=false,bool bAuditNativeLattice=false,
|
||||
bool bLatticeOrphanControl=false,bool bCaptureSkipWalkability=false,
|
||||
bool bProfilePassageComposer=false,bool bAllowUniformSkipsWhenHashing=false,
|
||||
bool bLinux77Profile=false)
|
||||
bool bLinux77Profile=false,int32 SchedulerPolicy=-1,
|
||||
bool bMovingCameraTrace=false,double MovingCameraSpeedMetersPerSecond=0.0,
|
||||
double MovingCameraDurationSeconds=12.0,double MovingCameraDwellSeconds=5.0)
|
||||
{
|
||||
using Clock=std::chrono::steady_clock;
|
||||
FStandaloneOwnerWorld& World=*OwnerWorld;
|
||||
@@ -2038,6 +2040,7 @@ namespace StandaloneStreamBenchmark
|
||||
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors,&Shape,bProfileColumnOverlap,
|
||||
FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache,
|
||||
bAllowUniformSkipsWhenHashing);
|
||||
Streamer.SetSchedulerPolicy(SchedulerPolicy);
|
||||
Streamer.SetUniformBlockProofAuditEnabled(bAuditUniformBlockProofs);
|
||||
Streamer.SetTileDrainInstrumentationEnabled(bInstrumentTileDrain);
|
||||
Streamer.SetLinux77ProfileEnabled(bLinux77Profile);
|
||||
@@ -4975,6 +4978,152 @@ namespace StandaloneStreamBenchmark
|
||||
std::fprintf(stderr,"[StreamPipeline] expected %zu tiles after shape publication but queued %d.\n",
|
||||
Desired.Tiles.size(),Update.DesiredTiles);return 7;
|
||||
}
|
||||
if(bMovingCameraTrace)
|
||||
{
|
||||
constexpr double SamplePeriodSeconds=0.25;
|
||||
using FTileSet=std::unordered_set<FVoxelTileKey,FStandaloneTileKeyHash>;
|
||||
FTileSet Resident;
|
||||
std::unordered_map<FVoxelTileKey,Clock::time_point,FStandaloneTileKeyHash> FarMissingSince;
|
||||
const double VoxelMetres=static_cast<double>(World.CoreVoxelSizeMicrons)/1000000.0;
|
||||
const auto TotalSeconds=MovingCameraDurationSeconds+MovingCameraDwellSeconds;
|
||||
const auto TraceStart=Clock::now();
|
||||
int32 SampleIndex=0;
|
||||
int32 GenerationFailures=0;
|
||||
double FirstAllFarReadySeconds=-1.0;
|
||||
int32 Samples=0;
|
||||
std::printf("[L79MovingCheckpoint] stage=start desired=%zu policy=%d speed_mps=%.3f\n",
|
||||
Desired.Tiles.size(),SchedulerPolicy,MovingCameraSpeedMetersPerSecond);
|
||||
std::fflush(stdout);
|
||||
auto DrainReady=[&]()
|
||||
{
|
||||
std::vector<FStandaloneCpuTile> Ready=Streamer.TakeReady();
|
||||
for(const FStandaloneCpuTile& Tile:Ready)
|
||||
{
|
||||
if(Tile.bGenerationFailed)++GenerationFailures;
|
||||
else Resident.insert(Tile.Key);
|
||||
}
|
||||
};
|
||||
for(;;++SampleIndex)
|
||||
{
|
||||
const double ScheduledSeconds=std::min(TotalSeconds,
|
||||
static_cast<double>(SampleIndex)*SamplePeriodSeconds);
|
||||
std::this_thread::sleep_until(TraceStart+std::chrono::duration<double>(ScheduledSeconds));
|
||||
const auto SampleAt=Clock::now();
|
||||
const double ElapsedSeconds=std::min(TotalSeconds,
|
||||
std::chrono::duration<double>(SampleAt-TraceStart).count());
|
||||
const double MovingSeconds=std::min(MovingCameraDurationSeconds,ElapsedSeconds);
|
||||
FVector Camera=CameraPositionVoxels;
|
||||
Camera.X+=static_cast<float>(MovingCameraSpeedMetersPerSecond*MovingSeconds/VoxelMetres);
|
||||
DrainReady();
|
||||
if(SampleIndex==0)
|
||||
{
|
||||
std::printf("[L79MovingCheckpoint] stage=drained resident=%zu failures=%d\n",
|
||||
Resident.size(),GenerationFailures);
|
||||
std::fflush(stdout);
|
||||
}
|
||||
FStandaloneDesiredTileSet CurrentDesired=Desired;
|
||||
CurrentDesired.CenterChunk=FIntVector(
|
||||
static_cast<int32>(std::floor(Camera.X/CHUNK_SIZE)),
|
||||
static_cast<int32>(std::floor(Camera.Y/CHUNK_SIZE)),
|
||||
static_cast<int32>(std::floor(Camera.Z/CHUNK_SIZE)));
|
||||
const auto CoreDesired=Shape.desiredTiles({Camera.X,Camera.Y,Camera.Z});
|
||||
CurrentDesired.Tiles.clear();CurrentDesired.Tiles.reserve(CoreDesired.size());
|
||||
for(const auto& Key:CoreDesired)
|
||||
{
|
||||
if(MaxLevel>=0&&Key.lod>MaxLevel)continue;
|
||||
CurrentDesired.Tiles.emplace_back(
|
||||
FIntVector(Key.coordinate.x,Key.coordinate.y,Key.coordinate.z),Key.lod);
|
||||
}
|
||||
const auto BeforeUpdate=Clock::now();
|
||||
const std::vector<FVoxelTileKey> ResidentKeys(Resident.begin(),Resident.end());
|
||||
const FStandaloneStreamingUpdate MovingUpdate=Streamer.UpdateDesiredWithSet(
|
||||
CurrentDesired,ResidentKeys,static_cast<uint64>(SampleIndex),Camera);
|
||||
if(SampleIndex==0)
|
||||
{
|
||||
std::printf("[L79MovingCheckpoint] stage=updated desired=%d new_jobs=%d\n",
|
||||
MovingUpdate.DesiredTiles,MovingUpdate.NewJobs);
|
||||
std::fflush(stdout);
|
||||
}
|
||||
const double UpdateMilliseconds=std::chrono::duration<double,std::milli>(
|
||||
Clock::now()-BeforeUpdate).count();
|
||||
if(MovingUpdate.DesiredTiles!=static_cast<int32>(CurrentDesired.Tiles.size()))
|
||||
{
|
||||
Streamer.Stop();
|
||||
std::fprintf(stderr,"[MovingCameraTrace] desired-set mismatch expected=%zu actual=%d\n",
|
||||
CurrentDesired.Tiles.size(),MovingUpdate.DesiredTiles);
|
||||
return 9;
|
||||
}
|
||||
std::size_t NearDesired=0,NearResident=0,FarDesired=0,FarResident=0,FarCovered=0;
|
||||
double FarStaleAgeSum=0.0,FarStaleAgeMax=0.0;
|
||||
const auto FloorDiv=[](int32 Value,int32 Divisor)
|
||||
{
|
||||
const int32 Quotient=Value/Divisor, Remainder=Value%Divisor;
|
||||
return Quotient-((Remainder!=0&&Remainder<0)?1:0);
|
||||
};
|
||||
std::unordered_map<FVoxelTileKey,Clock::time_point,FStandaloneTileKeyHash> CurrentFarMissing;
|
||||
for(const FVoxelTileKey& Key:CurrentDesired.Tiles)
|
||||
{
|
||||
const bool bResident=Resident.contains(Key);
|
||||
if(Key.Level<=2)
|
||||
{
|
||||
++NearDesired;
|
||||
NearResident+=bResident?1u:0u;
|
||||
}
|
||||
else
|
||||
{
|
||||
++FarDesired;
|
||||
FarResident+=bResident?1u:0u;
|
||||
bool bFarCovered=bResident;
|
||||
for(int32 ParentLOD=Key.Level+1;
|
||||
!bFarCovered&&ParentLOD<=World.MaxClipLevel;++ParentLOD)
|
||||
{
|
||||
const int32 Scale=1<<(ParentLOD-Key.Level);
|
||||
const FVoxelTileKey Parent(FIntVector(
|
||||
FloorDiv(Key.Coord.X,Scale),FloorDiv(Key.Coord.Y,Scale),
|
||||
FloorDiv(Key.Coord.Z,Scale)),ParentLOD);
|
||||
bFarCovered=Resident.contains(Parent);
|
||||
}
|
||||
FarCovered+=bFarCovered?1u:0u;
|
||||
if(!bResident)
|
||||
{
|
||||
auto [It,bInserted]=FarMissingSince.try_emplace(Key,SampleAt);
|
||||
(void)bInserted;
|
||||
const double Age=std::chrono::duration<double>(SampleAt-It->second).count();
|
||||
FarStaleAgeSum+=Age;
|
||||
FarStaleAgeMax=std::max(FarStaleAgeMax,Age);
|
||||
CurrentFarMissing.emplace(Key,It->second);
|
||||
}
|
||||
}
|
||||
}
|
||||
FarMissingSince.swap(CurrentFarMissing);
|
||||
const double NearReadyPercent=100.0*static_cast<double>(NearResident)
|
||||
/static_cast<double>(std::max<std::size_t>(1,NearDesired));
|
||||
const double FarMissingPercent=100.0*static_cast<double>(FarDesired-FarResident)
|
||||
/static_cast<double>(std::max<std::size_t>(1,FarDesired));
|
||||
const double FarHolePercent=100.0*static_cast<double>(FarDesired-FarCovered)
|
||||
/static_cast<double>(std::max<std::size_t>(1,FarDesired));
|
||||
const double FarStaleAgeMean=FarDesired>FarResident
|
||||
?FarStaleAgeSum/static_cast<double>(FarDesired-FarResident):0.0;
|
||||
if(FarDesired>0&&FarDesired==FarResident&&FirstAllFarReadySeconds<0.0)
|
||||
FirstAllFarReadySeconds=ElapsedSeconds;
|
||||
const char* Phase=ElapsedSeconds<MovingCameraDurationSeconds?"move":"dwell";
|
||||
std::printf("[L79MovingSample] policy=%d speed_mps=%.3f t_s=%.3f phase=%s x_m=%.3f near=%zu/%zu near_ready_pct=%.3f far=%zu/%zu far_missing_pct=%.3f far_covered=%zu/%zu far_hole_pct=%.3f far_stale_mean_s=%.3f far_stale_max_s=%.3f queued_jobs=%d queue_update_ms=%.3f generation_failures=%d\n",
|
||||
SchedulerPolicy,MovingCameraSpeedMetersPerSecond,ElapsedSeconds,Phase,
|
||||
Camera.X*VoxelMetres,NearResident,NearDesired,NearReadyPercent,
|
||||
FarResident,FarDesired,FarMissingPercent,FarCovered,FarDesired,
|
||||
FarHolePercent,FarStaleAgeMean,FarStaleAgeMax,
|
||||
MovingUpdate.NewJobs,UpdateMilliseconds,GenerationFailures);
|
||||
std::fflush(stdout);
|
||||
++Samples;
|
||||
if(ElapsedSeconds>=TotalSeconds)break;
|
||||
}
|
||||
std::printf("[L79MovingSummary] policy=%d speed_mps=%.3f move_s=%.3f dwell_s=%.3f samples=%d resident_tiles=%zu first_all_far_ready_s=%.3f generation_failures=%d priority=background_workers_only status=%s\n",
|
||||
SchedulerPolicy,MovingCameraSpeedMetersPerSecond,MovingCameraDurationSeconds,
|
||||
MovingCameraDwellSeconds,Samples,Resident.size(),FirstAllFarReadySeconds,
|
||||
GenerationFailures,GenerationFailures==0?"PASS":"FAIL");
|
||||
Streamer.Stop();
|
||||
return GenerationFailures==0?0:9;
|
||||
}
|
||||
FSceneExportWriter SceneExport;
|
||||
const bool bExportRequested=!MeshExportPath.empty();
|
||||
bool bSceneExportPassing=true;
|
||||
|
||||
@@ -551,6 +551,9 @@ public:
|
||||
bUseTileFilter=true;
|
||||
}
|
||||
|
||||
void SetSchedulerPolicy(int32 Policy)
|
||||
{SchedulerPolicy=std::clamp(Policy,-1,2);}
|
||||
|
||||
bool QueuePriorityTasks(std::vector<std::function<void()>> Tasks)
|
||||
{
|
||||
if(Tasks.empty())return true;
|
||||
@@ -623,6 +626,29 @@ public:
|
||||
return UpdateDesiredSet(std::move(NewDesired),ResidentKeys,CrossingIndex);
|
||||
}
|
||||
|
||||
FStandaloneStreamingUpdate UpdateDesiredWithSet(
|
||||
FStandaloneDesiredTileSet NewDesired,const std::vector<FVoxelTileKey>& ResidentKeys,
|
||||
uint64 CrossingIndex,const FVector& CameraPositionVoxels)
|
||||
{
|
||||
if(bHasLastCameraPosition)
|
||||
{
|
||||
const FVector Delta=CameraPositionVoxels-LastCameraPositionVoxels;
|
||||
const double LengthSquared=static_cast<double>(Delta.X)*Delta.X
|
||||
+static_cast<double>(Delta.Y)*Delta.Y;
|
||||
if(LengthSquared>1.0e-12)
|
||||
{
|
||||
const double InverseLength=1.0/std::sqrt(LengthSquared);
|
||||
TravelDirection=FVector(static_cast<float>(Delta.X*InverseLength),
|
||||
static_cast<float>(Delta.Y*InverseLength),0.0f);
|
||||
}
|
||||
else TravelDirection=FVector::ZeroVector;
|
||||
}
|
||||
else TravelDirection=FVector::ZeroVector;
|
||||
LastCameraPositionVoxels=CameraPositionVoxels;
|
||||
bHasLastCameraPosition=true;
|
||||
return UpdateDesiredSet(std::move(NewDesired),ResidentKeys,CrossingIndex);
|
||||
}
|
||||
|
||||
std::vector<FStandaloneCpuTile> TakeReady()
|
||||
{
|
||||
std::vector<FStandaloneCpuTile> Out;
|
||||
@@ -774,15 +800,18 @@ private:
|
||||
for(const auto& Pair:CancelledByCrossing)Stats.CancelledByCrossing.push_back(Pair);
|
||||
|
||||
std::vector<FVoxelTileKey> Prioritized=NewDesired.Tiles;
|
||||
if(IsCoarseFirstSchedulingEnabled())
|
||||
const int32 ActiveSchedulerPolicy=SchedulerPolicy<0
|
||||
?(IsCoarseFirstSchedulingEnabled()?2:3):SchedulerPolicy;
|
||||
if(ActiveSchedulerPolicy<=2)
|
||||
{
|
||||
// Finish the local walkable ring, then all near LODs, before admitting far work.
|
||||
// Terminer l'anneau praticable, puis tous les LOD proches avant le champ lointain.
|
||||
// Policy 0 retains L77 coarse-first ordering after the local support ring.
|
||||
// Policies 1 and 2 prioritize all near LODs before horizon work.
|
||||
auto Tier=[&](const FVoxelTileKey& Key)
|
||||
{
|
||||
if(Key.Level==0&&std::abs(Key.Coord.X-NewDesired.CenterChunk.X)<=1
|
||||
&&std::abs(Key.Coord.Y-NewDesired.CenterChunk.Y)<=1
|
||||
&&std::abs(Key.Coord.Z-NewDesired.CenterChunk.Z)<=1)return 0;
|
||||
if(ActiveSchedulerPolicy==0)return Key.Level>=3?1:2;
|
||||
return Key.Level<=2?1:2;
|
||||
};
|
||||
auto IsAhead=[&](const FVoxelTileKey& Key)
|
||||
@@ -798,8 +827,10 @@ private:
|
||||
{
|
||||
const int32 TierA=Tier(A),TierB=Tier(B);
|
||||
if(TierA!=TierB)return TierA<TierB;
|
||||
if(TierA==1&&A.Level!=B.Level)return A.Level<B.Level;
|
||||
if(TierA==2&&A.Level!=B.Level)return A.Level>B.Level;
|
||||
if(TierA==1&&A.Level!=B.Level)
|
||||
return ActiveSchedulerPolicy==0?A.Level>B.Level:A.Level<B.Level;
|
||||
if(TierA==2&&A.Level!=B.Level)
|
||||
return ActiveSchedulerPolicy==0?A.Level<B.Level:A.Level>B.Level;
|
||||
const bool bAheadA=IsAhead(A),bAheadB=IsAhead(B);
|
||||
if(bAheadA!=bAheadB)return bAheadA;
|
||||
const double DA=StandaloneWorldMesher::TileDistanceSquaredChunks(A,NewDesired.CenterChunk);
|
||||
@@ -810,6 +841,29 @@ private:
|
||||
if(A.Coord.Y!=B.Coord.Y)return A.Coord.Y<B.Coord.Y;
|
||||
return A.Coord.X<B.Coord.X;
|
||||
});
|
||||
|
||||
if(ActiveSchedulerPolicy==2)
|
||||
{
|
||||
// Give far LODs one queued slot per sixteen near requests. The near
|
||||
// prefix is the arrival burst; the recurring slot bounds horizon age.
|
||||
std::vector<FVoxelTileKey> NearTiles,FarTiles,FairOrder;
|
||||
NearTiles.reserve(Prioritized.size());
|
||||
FarTiles.reserve(Prioritized.size());
|
||||
FairOrder.reserve(Prioritized.size());
|
||||
for(const FVoxelTileKey& Key:Prioritized)
|
||||
(Key.Level<=2?NearTiles:FarTiles).push_back(Key);
|
||||
constexpr size_t NearBurstAndShareStride=16;
|
||||
size_t NearIndex=0,FarIndex=0;
|
||||
while(NearIndex<NearTiles.size())
|
||||
{
|
||||
const size_t NearEnd=std::min(NearTiles.size(),NearIndex+NearBurstAndShareStride);
|
||||
FairOrder.insert(FairOrder.end(),NearTiles.begin()+NearIndex,NearTiles.begin()+NearEnd);
|
||||
NearIndex=NearEnd;
|
||||
if(FarIndex<FarTiles.size())FairOrder.push_back(FarTiles[FarIndex++]);
|
||||
}
|
||||
FairOrder.insert(FairOrder.end(),FarTiles.begin()+FarIndex,FarTiles.end());
|
||||
Prioritized.swap(FairOrder);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -1028,13 +1082,16 @@ private:
|
||||
size_t DispatchQueueDepth=0,DispatchActiveWorkers=0;
|
||||
{
|
||||
std::unique_lock<std::mutex> Lock(Mutex);
|
||||
Wake.wait(Lock,[&](){return bStop||!PriorityTasks.empty()
|
||||
||(!Queue.empty()&&(NearOutstandingJobs==0||Queue.front().Key.Level<=2));});
|
||||
const auto HasEligibleTile=[&]()
|
||||
{
|
||||
return !Queue.empty()&&(SchedulerPolicy!=1||NearOutstandingJobs==0
|
||||
||Queue.front().Key.Level<=2);
|
||||
};
|
||||
Wake.wait(Lock,[&](){return bStop||!PriorityTasks.empty()||HasEligibleTile();});
|
||||
if(bStop)return;
|
||||
const bool bNearTileEligible=!Queue.empty()
|
||||
&&(NearOutstandingJobs==0||Queue.front().Key.Level<=2);
|
||||
const bool bTileEligible=HasEligibleTile();
|
||||
const bool bPreferPriority=!PriorityTasks.empty()
|
||||
&&(!bNearTileEligible||ConsecutivePriorityTasks<PriorityTaskBurst);
|
||||
&&(!bTileEligible||ConsecutivePriorityTasks<PriorityTaskBurst);
|
||||
if(bPreferPriority)
|
||||
{
|
||||
PriorityTask=std::move(PriorityTasks.front());PriorityTasks.pop_front();
|
||||
@@ -1192,6 +1249,7 @@ private:
|
||||
FVector TravelDirection=FVector::ZeroVector;
|
||||
bool bHasLastCameraPosition=false;
|
||||
int32 WorkerCount=8;
|
||||
int32 SchedulerPolicy=-1;
|
||||
bool bHashDensity=false;
|
||||
const FStandaloneMeshCache* MeshCache=nullptr;
|
||||
int32 MaxLevelFilter=-1;
|
||||
|
||||
Reference in new issue
Block a user