From 786ecc4cdfaaa4a41dbe9ade422e28fc3fa0e08e Mon Sep 17 00:00:00 2001 From: Fr0zka Date: Mon, 2 Nov 2026 16:45:20 +0100 Subject: [PATCH 1/4] streaming: bounded early admission - partial level sort, apply preflight, sorted transition index (L169, opt-in) With StreamingFastTravelWaitForAllCriticalTiles=0 and ApplyBudget=1 ms (test values; defaults stay 1 and 0), 7 pairs vs ca0bf76: near 43.3 -> 56.1%, forward 15.5 -> 17.9%, far holes 1.8 -> 0.0%, p99 7.06 -> 6.52 ms, worst 12.2 -> 11.8 ms, every run p99 <= 7.4 and worst <= 13.8 ms; arrival 1.671 -> 1.683 s (slower in 7/7) so not promoted. Source/: VoxelWorld.cpp, VoxelWorld.h. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1 --- Source/VoxelForge/Private/VoxelWorld.cpp | 181 ++++++- Source/VoxelForge/Public/VoxelWorld.h | 2 +- .../WorldAssemblerM3/Linux169/REPORT.md | 54 +++ .../Linux169/RunL169Movement.py | 442 ++++++++++++++++++ 4 files changed, 664 insertions(+), 15 deletions(-) create mode 100644 Standalone/Research/WorldAssemblerM3/Linux169/REPORT.md create mode 100644 Standalone/Research/WorldAssemblerM3/Linux169/RunL169Movement.py diff --git a/Source/VoxelForge/Private/VoxelWorld.cpp b/Source/VoxelForge/Private/VoxelWorld.cpp index 86138cc..8bf7021 100644 --- a/Source/VoxelForge/Private/VoxelWorld.cpp +++ b/Source/VoxelForge/Private/VoxelWorld.cpp @@ -7191,7 +7191,7 @@ void AVoxelWorld::LogReadyTransitionVisibilityMeasure() void AVoxelWorld::Tick(float DeltaTime) { const bool bTraceFrameTiming = VoxelForgeStartupTrace::IsActive(); - const uint64 TickStartCycles = bTraceFrameTiming ? FPlatformTime::Cycles64() : 0; + const uint64 TickStartCycles = FPlatformTime::Cycles64(); uint64 CollisionGateCycles = 0; uint64 UpdateChunksCycles = 0; uint64 PendingDrainCycles = 0; @@ -7418,7 +7418,7 @@ void AVoxelWorld::Tick(float DeltaTime) } const uint64 PendingDrainStartCycles = bTraceFrameTiming ? FPlatformTime::Cycles64() : 0; - ProcessPendingChunks(); + ProcessPendingChunks(TickStartCycles); if (bTraceFrameTiming) { PendingDrainCycles += FPlatformTime::Cycles64() - PendingDrainStartCycles; @@ -9899,7 +9899,7 @@ void AVoxelWorld::HandleTileCollisionCookComplete( } } -void AVoxelWorld::ProcessPendingChunks() +void AVoxelWorld::ProcessPendingChunks(uint64 TickStartCycles) { TRACE_CPUPROFILER_EVENT_SCOPE(VoxelForge_ProcessPending); // This runs on the game thread, called from Tick. @@ -10105,6 +10105,7 @@ void AVoxelWorld::ProcessPendingChunks() GetStreamingResultBacklogCount(), GVoxelForgeStreamingResultBacklogCap)); }; FChunkResult DequeuedChunk; + TArray DeferredApplyResults; const auto DequeueNextResult = [&]() -> bool { bDequeuedSupplementalFar = false; @@ -10234,8 +10235,6 @@ void AVoxelWorld::ProcessPendingChunks() const bool bRequestObsolete = DequeuedChunk.bObsolete || (Cancellation.IsValid() && Cancellation->bObsolete.load(std::memory_order_acquire)); - PendingTileCancellation.Remove(DequeuedChunk.Tile); - PendingTiles.Remove(DequeuedChunk.Tile); // A worker can observe shutdown after entering the mesher. Do not turn that partial // result into an all-air tile: it must remain eligible for a future generation epoch. @@ -10244,6 +10243,8 @@ void AVoxelWorld::ProcessPendingChunks() // if the tile leaves and re-enters the desired set before the queue is drained. if (DequeuedChunk.bAborted || bRequestObsolete) { + PendingTileCancellation.Remove(DequeuedChunk.Tile); + PendingTiles.Remove(DequeuedChunk.Tile); if (VoxelForgeStartupTrace::IsActive()) { VoxelForgeStartupTrace::RecordEvent(TEXT("tile_discard"), FString::Printf( @@ -10264,6 +10265,55 @@ void AVoxelWorld::ProcessPendingChunks() continue; } + // Estimate visible mesh submission cost before entering RMC's atomic CreateSectionGroup + // call. If this result would consume the remaining 11 ms tick target plus a 0.5 ms tail + // reserve, hold it for a later frame. The prediction is calibrated from L169 tile sizes: + // 2.2 ms fixed headroom + per-triangle and per-vertex work. Keep PendingTiles and its + // cancellation token alive while queued so the scheduler cannot submit a duplicate. + const bool bPreflightVisibleMesh = bFastTravelApplyBudgetActive + && DequeuedChunk.Epoch == GenerationEpoch + && IsDesired(DequeuedChunk.Tile) + && !DequeuedChunk.bEmpty + && DequeuedChunk.Streams.IsValid() + && DequeuedChunk.NumTriangles > 0; + if (bPreflightVisibleMesh) + { + const uint64 PreflightCycles = FPlatformTime::Cycles64(); + const double TickElapsedMilliseconds = TickStartCycles != 0 + && PreflightCycles >= TickStartCycles + ? FPlatformTime::ToSeconds64(PreflightCycles - TickStartCycles) * 1000.0 + : 0.0; + const double EstimatedApplyMilliseconds = 2.2 + + static_cast(FMath::Max(0, DequeuedChunk.NumTriangles)) * 0.00012 + + static_cast(FMath::Max(0, DequeuedChunk.NumVertices)) * 0.00003; + constexpr double TickApplyTargetMilliseconds = 11.0; + constexpr double TickApplyTailReserveMilliseconds = 0.5; + const double RemainingApplyMilliseconds = FMath::Max( + 0.0, TickApplyTargetMilliseconds - TickApplyTailReserveMilliseconds + - TickElapsedMilliseconds); + if (EstimatedApplyMilliseconds > RemainingApplyMilliseconds) + { + if (VoxelForgeStartupTrace::IsActive()) + { + VoxelForgeStartupTrace::RecordEvent(TEXT("tile_apply_deferred"), FString::Printf( + TEXT("\"frame\":%llu,\"tile\":[%d,%d,%d],\"level\":%d," + "\"triangles\":%d,\"vertices\":%d,\"estimated_apply_ms\":%.3f," + "\"elapsed_tick_ms\":%.3f,\"remaining_apply_ms\":%.3f"), + static_cast(GFrameCounter), + DequeuedChunk.Tile.Coord.X, DequeuedChunk.Tile.Coord.Y, + DequeuedChunk.Tile.Coord.Z, DequeuedChunk.Tile.Level, + DequeuedChunk.NumTriangles, DequeuedChunk.NumVertices, + EstimatedApplyMilliseconds, TickElapsedMilliseconds, + RemainingApplyMilliseconds)); + } + DeferredApplyResults.Add(MoveTemp(DequeuedChunk)); + continue; + } + } + + PendingTileCancellation.Remove(DequeuedChunk.Tile); + PendingTiles.Remove(DequeuedChunk.Tile); + // ApplyTileResult does epoch check, capture ingest, empty-release / mesh submission, and // only then records the tile as loaded. Only a real (visible) upload counts against the // per-frame budget — stale/empty drains are free. @@ -10323,6 +10373,11 @@ void AVoxelWorld::ProcessPendingChunks() } } + for (FChunkResult& DeferredResult : DeferredApplyResults) + { + EnqueueStreamingResult(MoveTemp(DeferredResult)); + } + if (bProfileApply && ResultsDrained > 0) { UE_LOG(LogTemp, Display, @@ -10827,7 +10882,11 @@ void AVoxelWorld::HideOverlappingDesiredVisuals(const FVoxelTileKey& RetiredTile TArray> OverlappingDesired; if (const TArray* Descendants = DesiredTransitionDescendants.Find(RetiredTile)) { - OverlappingDesired.Append(*Descendants); + OverlappingDesired.Reserve(OverlappingDesired.Num() + Descendants->Num()); + for (const FVoxelTileKey& Descendant : *Descendants) + { + OverlappingDesired.Add(Descendant); + } } for (int32 Level = RetiredTile.Level + 1; Level <= DesiredTransitionMaxLevel; ++Level) { @@ -11064,9 +11123,18 @@ static FORCEINLINE void VF_OuterShell(const UVoxelSettings* Settings, int32 R, i VoxelClipmapDesiredTiles::OuterShell(Parameters, R, MaxLevel, OutLevel, OutRadius); } +static bool VF_TransitionIndexKeyLess(const FVoxelTileKey& A, const FVoxelTileKey& B) +{ + if (A.Level != B.Level) return A.Level < B.Level; + if (A.Coord.X != B.Coord.X) return A.Coord.X < B.Coord.X; + if (A.Coord.Y != B.Coord.Y) return A.Coord.Y < B.Coord.Y; + return A.Coord.Z < B.Coord.Z; +} + void AVoxelWorld::BuildDesiredTransitionIndex() { DesiredTransitionDescendants.Reset(); + DesiredTransitionDescendants.Reserve(DesiredSorted.Num()); DesiredTransitionMaxLevel = 0; for (const FVoxelTileKey& Key : DesiredSorted) { @@ -11079,7 +11147,9 @@ void AVoxelWorld::BuildDesiredTransitionIndex() constexpr int32 MaxSupportedTileLevel = 8; const int32 DescendantIndexMaxLevel = FMath::Max( DesiredTransitionMaxLevel, MaxSupportedTileLevel); - for (const FVoxelTileKey& Key : DesiredSorted) + TArray TransitionIndexBuildKeys = DesiredSorted; + TransitionIndexBuildKeys.Sort(VF_TransitionIndexKeyLess); + for (const FVoxelTileKey& Key : TransitionIndexBuildKeys) { for (int32 Level = Key.Level + 1; Level <= DescendantIndexMaxLevel; ++Level) { @@ -11087,7 +11157,12 @@ void AVoxelWorld::BuildDesiredTransitionIndex() VF_AppendTouchingAncestors(Key, Level, TouchingAncestors); for (const FVoxelTileKey& Ancestor : TouchingAncestors) { - DesiredTransitionDescendants.FindOrAdd(Ancestor).Add(Key); + TArray& Descendants = + DesiredTransitionDescendants.FindOrAdd(Ancestor); + if (Descendants.IsEmpty() || Descendants.Last() != Key) + { + Descendants.Add(Key); + } } } } @@ -11187,6 +11262,10 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& double OverlayMaterializationMilliseconds = 0.0; double PrioritySortMilliseconds = 0.0; double CriticalAdmissionOrderMilliseconds = 0.0; + double DesiredCollisionAndStampMilliseconds = 0.0; + double DesiredTransitionIndexMilliseconds = 0.0; + double DesiredRetainedRegionMilliseconds = 0.0; + double DesiredRegionReconcileMilliseconds = 0.0; int32 IncrementalChangedLod4PlusKeyCount = 0; int32 IncrementalFallbackReason = 0; bool bCoreDesiredCacheHit = false; @@ -11954,6 +12033,8 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& (FPlatformTime::Seconds() - PrioritySortStartSeconds) * 1000.0; } + const double DesiredCollisionAndStampStartSeconds = bTraceDesiredBreakdown + ? FPlatformTime::Seconds() : 0.0; RestoreKnownEmptyCollisionTiles(); ReconcileCollisionDesiredTiles(); for (auto It = DesiredStamped.CreateIterator(); It; ++It) @@ -11965,6 +12046,11 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& } } CancelObsoleteTileWork(); + if (DesiredCollisionAndStampStartSeconds != 0.0) + { + DesiredCollisionAndStampMilliseconds = + (FPlatformTime::Seconds() - DesiredCollisionAndStampStartSeconds) * 1000.0; + } const double CriticalAdmissionOrderStartSeconds = bTraceDesiredBreakdown ? FPlatformTime::Seconds() : 0.0; @@ -12135,6 +12221,8 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& checkf(false, TEXT("VoxelForge incremental desired priority-order equality failure")); } } + const double DesiredTransitionIndexStartSeconds = bTraceDesiredBreakdown + ? FPlatformTime::Seconds() : 0.0; if (bIncrementalDesiredSet && bHadPreviousDesiredSet) { constexpr int32 MaxSupportedTileLevel = 8; @@ -12143,6 +12231,27 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& const auto ApplyTransitionDelta = [this, TransitionIndexMaxLevel]( const FVoxelTileKey& Key, bool bAdd) { + const auto LowerBound = [](const TArray& Descendants, + const FVoxelTileKey& SearchKey) + { + int32 First = 0; + int32 Count = Descendants.Num(); + while (Count > 0) + { + const int32 Step = Count / 2; + const int32 Index = First + Step; + if (VF_TransitionIndexKeyLess(Descendants[Index], SearchKey)) + { + First = Index + 1; + Count -= Step + 1; + } + else + { + Count = Step; + } + } + return First; + }; for (int32 Level = Key.Level + 1; Level <= TransitionIndexMaxLevel; ++Level) { TArray> TouchingAncestors; @@ -12153,12 +12262,21 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& { TArray& Descendants = DesiredTransitionDescendants.FindOrAdd(Ancestor); - if (!Descendants.Contains(Key)) Descendants.Add(Key); + const int32 InsertIndex = LowerBound(Descendants, Key); + if (InsertIndex == Descendants.Num() || Descendants[InsertIndex] != Key) + { + Descendants.Insert(Key, InsertIndex); + } } else if (TArray* Descendants = DesiredTransitionDescendants.Find(Ancestor)) { - Descendants->RemoveSingleSwap(Key, EAllowShrinking::No); + const int32 RemoveIndex = LowerBound(*Descendants, Key); + if (RemoveIndex < Descendants->Num() + && (*Descendants)[RemoveIndex] == Key) + { + Descendants->RemoveAt(RemoveIndex, 1, EAllowShrinking::No); + } if (Descendants->IsEmpty()) DesiredTransitionDescendants.Remove(Ancestor); } } @@ -12171,7 +12289,14 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& { BuildDesiredTransitionIndex(); } + if (DesiredTransitionIndexStartSeconds != 0.0) + { + DesiredTransitionIndexMilliseconds = + (FPlatformTime::Seconds() - DesiredTransitionIndexStartSeconds) * 1000.0; + } + const double DesiredRetainedRegionStartSeconds = bTraceDesiredBreakdown + ? FPlatformTime::Seconds() : 0.0; bool bRetainedSetChanged = false; int32 RetainedTileCount = 0; TArray MissingRetainedFineTiles; @@ -12190,6 +12315,13 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& } } } + if (DesiredRetainedRegionStartSeconds != 0.0) + { + DesiredRetainedRegionMilliseconds = + (FPlatformTime::Seconds() - DesiredRetainedRegionStartSeconds) * 1000.0; + } + const double DesiredRegionReconcileStartSeconds = bTraceDesiredBreakdown + ? FPlatformTime::Seconds() : 0.0; TArray MissingRegionTiles; { FReadScopeLock Lock(State->WorldLock); @@ -12249,6 +12381,11 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& } StartPendingCoreRegionCompile(); } + if (DesiredRegionReconcileStartSeconds != 0.0) + { + DesiredRegionReconcileMilliseconds = + (FPlatformTime::Seconds() - DesiredRegionReconcileStartSeconds) * 1000.0; + } if (bTraceDesiredBreakdown) { @@ -12258,7 +12395,10 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"total_ms\":%s," "\"core_prepare_ms\":%s,\"core_delta_ms\":%s,\"fallback_ms\":%s," "\"overlay_materialize_ms\":%s,\"priority_sort_ms\":%s," - "\"critical_order_ms\":%s,\"core_keys\":%d,\"candidate_keys\":%d," + "\"critical_order_ms\":%s,\"collision_stamp_ms\":%s," + "\"transition_index_ms\":%s,\"retained_region_ms\":%s," + "\"region_reconcile_ms\":%s," + "\"core_keys\":%d,\"candidate_keys\":%d," "\"overlay_keys\":%d,\"critical_keys\":%d,\"fallback_reason\":%d," "\"changed_lod4plus\":%d,\"materialization_update\":%d," "\"priority_order_reused\":%d,\"priority_order_updated\":%d," @@ -12271,6 +12411,10 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector& *FString::SanitizeFloat(static_cast(OverlayMaterializationMilliseconds)), *FString::SanitizeFloat(static_cast(PrioritySortMilliseconds)), *FString::SanitizeFloat(static_cast(CriticalAdmissionOrderMilliseconds)), + *FString::SanitizeFloat(static_cast(DesiredCollisionAndStampMilliseconds)), + *FString::SanitizeFloat(static_cast(DesiredTransitionIndexMilliseconds)), + *FString::SanitizeFloat(static_cast(DesiredRetainedRegionMilliseconds)), + *FString::SanitizeFloat(static_cast(DesiredRegionReconcileMilliseconds)), CoreDesiredTiles ? CoreDesiredTiles->Num() : static_cast(UncachedCoreDesiredTiles.size()), CandidateKeys.Num(), OverlayDesiredTiles.Num(), CriticalDesiredTiles.Num(), @@ -14118,14 +14262,17 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw { const uint64 TraceLevelSortStartCycles = bTraceStreamingScheduler ? FPlatformTime::Cycles64() : 0; + const int32 LevelSubmissionCount = FMath::Min( + LevelTiles.Num(), FMath::Max( + 0, AdmissionCap - GetGenerationAdmissionCount())); const bool bUseForwardFirstOrder = GVoxelForgeStreamingFastTravelForwardFirst != 0 && bPolicy3FastTravel && Level <= 2; - if (bUseForwardFirstOrder) + if (LevelSubmissionCount > 0 && bUseForwardFirstOrder) { SortNearTilesForwardFirst(LevelTiles); } - else + else if (LevelSubmissionCount > 0) { FVector Heading = GetActorTransform().InverseTransformVectorNoScale(PlayerHeading); if (Heading.IsNearlyZero() && IsValid(PlayerPawn)) @@ -14138,7 +14285,12 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw const double TileSizeCm = static_cast(VF_ClipLevelExtentVoxels(Level)) * GetActiveVoxelSizeCm(); const bool bUseFastNearLeadOrder = bPolicy3FastTravel && Level <= 2; - std::stable_sort(LevelTiles.GetData(), LevelTiles.GetData() + LevelTiles.Num(), + // The full order is deterministic because the comparator breaks every tie + // with the tile coordinates. Select only the prefix this admission pass can + // submit; sorting every outstanding key made a small worker cap pay O(N log N). + std::partial_sort(LevelTiles.GetData(), + LevelTiles.GetData() + LevelSubmissionCount, + LevelTiles.GetData() + LevelTiles.Num(), [&](const FVoxelTileKey& A, const FVoxelTileKey& B) { const FVector CenterA( @@ -14193,6 +14345,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw if (A.Coord.Y != B.Coord.Y) return A.Coord.Y < B.Coord.Y; return A.Coord.Z < B.Coord.Z; }); + LevelTiles.SetNum(LevelSubmissionCount, EAllowShrinking::No); } if (bTraceStreamingScheduler) { diff --git a/Source/VoxelForge/Public/VoxelWorld.h b/Source/VoxelForge/Public/VoxelWorld.h index 0a0078c..9b8cbb1 100644 --- a/Source/VoxelForge/Public/VoxelWorld.h +++ b/Source/VoxelForge/Public/VoxelWorld.h @@ -1617,7 +1617,7 @@ public: // ONE frame → a game-thread teardown spike. Drained by ProcessUnloadQueue (catch-up scaled). TSet PendingUnload; - void ProcessPendingChunks(); + void ProcessPendingChunks(uint64 TickStartCycles); void ProcessUnloadQueue(); // budgeted teardown drain (see PendingUnload) void QueuePendingUnload(const FVoxelTileKey& Tile); void HideOverlappingDesiredVisuals(const FVoxelTileKey& RetiredTile); diff --git a/Standalone/Research/WorldAssemblerM3/Linux169/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux169/REPORT.md new file mode 100644 index 0000000..2fdcf21 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux169/REPORT.md @@ -0,0 +1,54 @@ +# L169 — Early admission stays opt-in + +**Result:** Do not promote L157 early admission. Six of seven promotion gates passed, but candidate LOD0–2 arrival was slower in every pair: median **1.683 s** versus **1.671 s** on `assembler/world` `ca0bf76` (+12.4 ms). The source defaults are restored to wait-for-critical **1** and apply-budget **0 ms**. The candidate settings remain explicit overrides in the runner. + +## Measurement + +Seven interleaved W8 pairs used the four-strata owner plan, 25 cm core voxels, 30 m/s movement, a 12-second move from t=6 to t=18, and a 5-second dwell. Each run had about 46 moving samples; one warm-up per binary was excluded. No product CVar was changed except the two L157 switches under test. At measurement time, the candidate binary compiled those settings as defaults; the runner now supplies the same values explicitly so the experiment can be repeated with shipping defaults off. + +| Moving-window metric (median across seven runs) | `ca0bf76` | Candidate | Result | +|---|---:|---:|---:| +| Tick p99 | 7.060 ms | 6.519 ms | lower | +| Worst tick | 12.219 ms | 11.776 ms | lower | +| Near LOD0–2 readiness | 43.28% | 56.12% | +12.84 pp | +| Forward-cone readiness | 15.52% | 17.93% | +2.41 pp | +| Far holes | 1.80% | 0.00% | lower | +| First settled LOD0–2 | 1.671 s | 1.683 s | **12.4 ms slower** | + +Every run stayed below 11 ms p99 and 14 ms worst tick. The largest observed p99 was 7.400 ms; the largest worst tick was 13.776 ms. Candidate worst-tick median was not worse. Arrival remained slower after subtracting each run’s first streaming sample (1.305 s vs 1.299 s), so the raw arrival failure is not explained entirely by trace start time. + +## Spike count and changes + +All counts below use the 12-second moving window unless identified as a startup diagnostic. + +- **Level sorting:** With L157 on at the base defaults, the largest full-sort sample was 0.865 ms and no sample exceeded 1 ms; the L167 8.3 ms spike did not recur. Candidate partial sorting reduced the largest sample to 0.123 ms, with no sample above 1 ms. +- **Desired rebuild and overlay:** Base per-run desired-rebuild peaks had a 7.476 ms median and reached 8.110 ms. Candidate peaks had a 6.928 ms median and reached 7.260 ms; rebuilds over 5 ms still occurred (median four per run, maximum five). Overlay materialization never exceeded 2 ms; its candidate maximum was 1.094 ms. The transition descendant index now uses sorted arrays and binary-search deltas. In a separate pre-move startup trace with 2,233 Core keys, the initial index build took 1.605 ms; the maximum moving-window index phase was 3.626 ms. +- **Mesh apply:** The 2.8 ms historical apply spike did not recur. Maximum per-frame apply time fell from 2.548 ms at base to 1.590 ms candidate; largest individual candidate apply was 1.512 ms. The preflight deferred 21–145 results per run (median 109), with a largest estimate of 3.447 ms for 8,979 triangles. RealtimeMesh submission remains atomic, so the estimate is a heuristic and does not split a single upload. + +The cold-start arrival gate prevents default promotion even though movement readiness and tick behavior improved. No further adjustment was made after the seven-pair result. + +## Defaults and checks + +| CVar | Final default | Candidate test value | +|---|---:|---:| +| `voxel.StreamingFastTravelWaitForAllCriticalTiles` | 1 | 0 | +| `voxel.StreamingFastTravelApplyBudgetMilliseconds` | 0 ms | 1 ms | +| `voxel.StreamingIncrementalDesiredSet` | 1 | 1 | +| `voxel.StreamingFastTravelForwardFirst` | 0 | 0 | +| `voxel.StreamingRebuildOnCoreRegionCompletion` | 1 | 1 | +| `voxel.StreamingCoreDesiredCacheByCenter` | 0 | 0 | +| `voxel.StreamingIncrementalDesiredAuditEvery` | 0 | 0 | +| `voxel.StreamingIncrementalDesiredMaterializationAuditEvery` | 0 | 0 | +| `voxel.StreamingIncrementalDesiredAuditInjectMismatch` | 0 | 0 | + +- Final VFHost Linux Development build passed with `-NoUBA -MaxParallelActions=4`; three existing deprecated RealtimeMesh API warnings remain. +- Final-source `verify.py fast` passed in 289.8 s: 268 passes, 0 failures, 5 diagnostics. Clang and GCC geometry hashes and W1/W8/W12 determinism passed. +- `git diff --check` and the runner AST syntax check passed. The direct live `TileHashDump` hook returned zero records for the Core path on both binaries and is inconclusive; the fast-tier geometry and determinism checks passed. +- UE W12 movement was not run: this VFHost procedure exposed eight effective tile slots. Fast-tier W12 determinism was run. + +Remaining risk: desired rebuilds still reach 7.260 ms in the candidate, and the apply estimate cannot bound an atomic RMC upload. Most importantly, the arrival gate failed, so early admission remains opt-in. + +Changed Unreal `Source/` files: + +- `Source/VoxelForge/Private/VoxelWorld.cpp` +- `Source/VoxelForge/Public/VoxelWorld.h` diff --git a/Standalone/Research/WorldAssemblerM3/Linux169/RunL169Movement.py b/Standalone/Research/WorldAssemblerM3/Linux169/RunL169Movement.py new file mode 100644 index 0000000..2178c38 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux169/RunL169Movement.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""L169 default-based UE movement comparison and early-admission spike trace.""" +from __future__ import annotations + +import argparse +import json +import math +import shutil +import statistics +import subprocess +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[4] +HOST = Path("/home/jahni/VFHost") +EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd") +HOST_PLUGIN = HOST / "Binaries/Linux/libUnrealEditor-VoxelForge.so" +OUT = ROOT / "Saved/Linux/l169/ue/movement" +BIN_DIR = ROOT / "Saved/Linux/l169/bin" +SPAWN_M = (19.166015625, -14.2001953125, -753.3740234375) +MOVE_START_S = 6.0 +MOVE_DURATION_S = 12.0 + +BASELINE_BINARY = BIN_DIR / "ca0bf76/libUnrealEditor-VoxelForge.so" +CANDIDATE_BINARY = BIN_DIR / "early-admission/libUnrealEditor-VoxelForge.so" +DIAGNOSTIC_BINARY = BIN_DIR / "apply-preflight/libUnrealEditor-VoxelForge.so" + +EXPECTED = { + "baseline": {"wait": 1, "apply_budget_ms": 0.0, "incremental": 1}, + "early_override": {"wait": 0, "apply_budget_ms": 1.0, "incremental": 1}, + "candidate": {"wait": 0, "apply_budget_ms": 1.0, "incremental": 1}, +} + +ORDER = { + 1: ("baseline", "candidate"), + 2: ("candidate", "baseline"), + 3: ("baseline", "candidate"), + 4: ("candidate", "baseline"), + 5: ("baseline", "candidate"), + 6: ("candidate", "baseline"), + 7: ("baseline", "candidate"), +} + + +def percentile(values: list[float], fraction: float = 0.99) -> float | None: + if not values: + return None + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1)) + return ordered[index] + + +def percent(numerator: int | float, denominator: int | float) -> float | None: + return 100.0 * numerator / denominator if denominator > 0 else None + + +def read_events(path: Path) -> list[dict]: + rows = [] + if path.is_file(): + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not line.startswith("{"): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def summarize(condition: str, run_name: str, trace: Path, log: Path, exit_code: int, + wall_s: float, audit_or_diag: bool = False) -> dict: + expected = EXPECTED[condition] + events = read_events(trace) + backend = next((e for e in events if e.get("type") == "worker_backend_config"), {}) + scheduler_all = [e for e in events if e.get("type") == "scheduler_admission"] + if not scheduler_all: + raise RuntimeError(f"missing scheduler readback: {trace}") + if any(int(e.get("incremental_desired_set", -1)) != expected["incremental"] + for e in scheduler_all): + raise RuntimeError(f"incremental desired-set readback mismatch: {trace}") + if any(int(e.get("forward_first", -1)) != 0 + or int(e.get("rebuild_on_core_region_completion", -1)) != 1 + or int(e.get("core_desired_cache_by_center", -1)) != 0 + or int(e.get("incremental_desired_audit_every", -1)) != 0 + or int(e.get("incremental_desired_materialization_audit_every", -1)) != 0 + or int(e.get("incremental_desired_audit_inject_mismatch", -1)) != 0 + for e in scheduler_all): + raise RuntimeError(f"non-default L160/audit CVar readback: {trace}") + if (int(backend.get("streaming_fast_travel_wait_for_all_critical_tiles", -1)) + != expected["wait"] + or abs(float(backend.get("streaming_fast_travel_apply_budget_ms", -999.0)) + - expected["apply_budget_ms"]) > 0.01 + or int(backend.get("effective_tile_slots", -1)) != 8): + raise RuntimeError(f"early-admission/worker readback mismatch: {trace}; {backend}") + + move_stop = next((e for e in events if e.get("type") == "streaming_test_move_stopped"), None) + if move_stop is None: + raise RuntimeError(f"missing move-stop event: {trace}") + stop_s = float(move_stop.get("t_s", 0.0)) + samples = [e for e in events if e.get("type") == "streaming_sample"] + moving = [e for e in samples if MOVE_START_S <= float(e.get("t_s", 0.0)) < stop_s + and float(e.get("speed_cm_s", 0.0)) >= 50.0] + dwell = [e for e in samples if float(e.get("t_s", 0.0)) >= stop_s + and float(e.get("speed_cm_s", 0.0)) < 50.0] + if exit_code != 0 or len(moving) < 40 or len(dwell) < 16: + raise RuntimeError(f"incomplete {run_name}: exit={exit_code}, move={len(moving)}, " + f"dwell={len(dwell)}") + move_start = min(float(e.get("t_s", 0.0)) for e in moving) + + work = [e for e in events if e.get("type") == "game_thread_work_frame" + and move_start <= float(e.get("t_s", 0.0)) < stop_s] + scheduler_moving = [e for e in scheduler_all + if move_start <= float(e.get("t_s", 0.0)) < stop_s] + rebuild_events = [e for e in events if e.get("type") == "desired_rebuild_breakdown" + and move_start <= float(e.get("t_s", 0.0)) < stop_s] + applies = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "apply_frame" + and move_start <= float(e.get("t_s", 0.0)) < stop_s} + scheduler = {int(e.get("frame", -1)): e for e in scheduler_all + if move_start <= float(e.get("t_s", 0.0)) < stop_s} + desired = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "desired_rebuild_breakdown" + and move_start <= float(e.get("t_s", 0.0)) < stop_s} + moving_tiles = [e for e in events if e.get("type") == "tile" + and move_start <= float(e.get("t_s", 0.0)) < stop_s] + apply_deferrals = [e for e in events if e.get("type") == "tile_apply_deferred" + and move_start <= float(e.get("t_s", 0.0)) < stop_s] + + frames = [] + for event in work: + frame = int(event.get("frame", -1)) + apply = applies.get(frame, {}) + admit = scheduler.get(frame, {}) + rebuild = desired.get(frame, {}) + frames.append({ + "frame": frame, + "tick_ms": float(event.get("tick_ms", 0.0)), + "update_chunks_ms": float(event.get("update_chunks_ms", 0.0)), + "pending_drain_ms": float(event.get("pending_drain_ms", 0.0)), + "apply_ms": float(apply.get("apply_ms", 0.0)), + "apply_budget_ms": float(apply.get("apply_budget_ms", 0.0)), + "meshes_applied": int(apply.get("meshes_applied", 0)), + "time_budget_hit": int(apply.get("time_limit_reached", 0)), + "desired_rebuild_ms": float(admit.get("desired_rebuild_ms", 0.0)), + "desired_rebuild_event_ms": float(rebuild.get("total_ms", 0.0)), + "overlay_materialize_ms": float(rebuild.get("overlay_materialize_ms", 0.0)), + "core_prepare_ms": float(rebuild.get("core_prepare_ms", 0.0)), + "core_delta_ms": float(rebuild.get("core_delta_ms", 0.0)), + "priority_sort_ms": float(rebuild.get("priority_sort_ms", 0.0)), + "critical_order_ms": float(rebuild.get("critical_order_ms", 0.0)), + "collision_stamp_ms": float(rebuild.get("collision_stamp_ms", 0.0)), + "transition_index_ms": float(rebuild.get("transition_index_ms", 0.0)), + "retained_region_ms": float(rebuild.get("retained_region_ms", 0.0)), + "region_reconcile_ms": float(rebuild.get("region_reconcile_ms", 0.0)), + "level_sort_ms": float(admit.get("level_sort_ms", 0.0)), + "level_candidate_scan_ms": float(admit.get("level_candidate_scan_ms", 0.0)), + "submit_tiles_ms": float(admit.get("submit_tiles_ms", 0.0)), + }) + + near = [] + cone = [] + far_holes = [] + for event in moving: + desired_by_lod = event.get("desired_by_lod", [0] * 9) + resident_by_lod = event.get("resident_by_lod", [0] * 9) + covered_by_lod = event.get("covered_by_lod", resident_by_lod) + near.append(percent(sum(resident_by_lod[:3]), sum(desired_by_lod[:3])) or 0.0) + cone.append(percent(int(event.get("near_forward_cone_resident", 0)), + int(event.get("near_forward_cone_desired", 0))) or 0.0) + far_holes.append(percent(sum(desired_by_lod[3:]) - sum(covered_by_lod[3:]), + sum(desired_by_lod[3:])) or 0.0) + settled = next((e for e in samples if float(e.get("t_s", 0.0)) < move_start + and float(e.get("speed_cm_s", 0.0)) < 50.0 + and sum(e.get("settled_by_lod", [0] * 9)[:3]) + >= sum(e.get("desired_by_lod", [0] * 9)[:3]) > 0), None) + p99 = percentile([r["tick_ms"] for r in frames]) + worst = max(frames, key=lambda r: r["tick_ms"], default={}) + row = { + "condition": condition, + "run": run_name, + "diagnostic_override": audit_or_diag, + "source_default_incremental": expected["incremental"], + "early_admission_wait_for_critical": expected["wait"], + "early_admission_apply_budget_ms": expected["apply_budget_ms"], + "speed_mps": 30, + "effective_tile_slots": backend.get("effective_tile_slots"), + "moving_samples": len(moving), + "dwell_samples": len(dwell), + "tick_p99_ms": p99, + "tick_worst_ms": worst.get("tick_ms"), + "worst_frame": worst, + "near_readiness_moving_mean_pct": statistics.mean(near) if near else None, + "forward_cone_readiness_moving_mean_pct": statistics.mean(cone) if cone else None, + "far_holes_moving_median_pct": statistics.median(far_holes) if far_holes else None, + "arrival_lod02_settled_s": float(settled.get("t_s", 0.0)) if settled else None, + "desired_rebuild_count": len(rebuild_events), + "incremental_materialization_updates": sum( + int(e.get("materialization_update", 0)) != 0 for e in events + if e.get("type") == "desired_rebuild_breakdown" + and move_start <= float(e.get("t_s", 0.0)) < stop_s), + "core_incremental_deltas": sum( + int(e.get("incremental_delta", 0)) != 0 for e in events + if e.get("type") == "core_desired_cache" + and move_start <= float(e.get("t_s", 0.0)) < stop_s), + "level_sort_samples_over_1ms": sum( + float(e.get("level_sort_ms", 0.0)) > 1.0 for e in scheduler_moving), + "level_sort_max_ms": max((float(e.get("level_sort_ms", 0.0)) + for e in scheduler_moving), default=0.0), + "desired_rebuild_events_over_5ms": sum( + float(e.get("total_ms", 0.0)) > 5.0 for e in rebuild_events), + "desired_rebuild_max_ms": max( + (float(e.get("total_ms", 0.0)) for e in rebuild_events), default=0.0), + "overlay_materialize_events_over_2ms": sum( + float(e.get("overlay_materialize_ms", 0.0)) > 2.0 for e in rebuild_events), + "overlay_materialize_max_ms": max( + (float(e.get("overlay_materialize_ms", 0.0)) for e in rebuild_events), default=0.0), + "desired_subphase_max_ms": { + key: max((float(e.get(key, 0.0)) for e in rebuild_events), default=0.0) + for key in ("core_prepare_ms", "overlay_materialize_ms", "priority_sort_ms", + "critical_order_ms", "collision_stamp_ms", "transition_index_ms", + "retained_region_ms", "region_reconcile_ms")}, + "apply_budget_hit_frames": sum(f["time_budget_hit"] for f in frames), + "apply_frame_count": sum(f["meshes_applied"] > 0 for f in frames), + "apply_ms_max": max((f["apply_ms"] for f in frames), default=0.0), + "tiles_over_1ms_apply": sum(float(e.get("apply_s", 0.0)) > 0.001 + for e in moving_tiles), + "apply_preflight_deferred_count": len(apply_deferrals), + "apply_preflight_deferred_max_estimate_ms": max( + (float(e.get("estimated_apply_ms", 0.0)) for e in apply_deferrals), default=0.0), + "apply_preflight_deferred_max_triangles": max( + (int(e.get("triangles", 0)) for e in apply_deferrals), default=0), + "largest_tile_applies": sorted(({ + "apply_ms": float(e.get("apply_s", 0.0)) * 1000.0, + "triangles": int(e.get("triangles", 0)), + "vertices": int(e.get("vertices", 0)), + "level": int(e.get("level", -1)), + } for e in moving_tiles), + key=lambda e: e["apply_ms"], reverse=True)[:8], + "top_ticks": sorted(frames, key=lambda r: r["tick_ms"], reverse=True)[:8], + "trace": str(trace.relative_to(ROOT)), + "log": str(log.relative_to(ROOT)), + "wall_s": round(wall_s, 3), + } + return row + + +def run(condition: str, run_name: str, diagnostic_override: bool = False, + tile_hash_dump: Path | None = None) -> dict: + if condition == "baseline": + binary = BASELINE_BINARY + elif condition == "early_override": + binary = DIAGNOSTIC_BINARY if DIAGNOSTIC_BINARY.is_file() else BASELINE_BINARY + elif condition == "candidate": + binary = CANDIDATE_BINARY + else: + raise ValueError(condition) + if not binary.is_file(): + raise FileNotFoundError(binary) + HOST_PLUGIN.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(binary, HOST_PLUGIN) + out = OUT / condition + export = HOST / "Plugins/VoxelForge/Saved/Linux/l169" / condition / run_name + userdir = HOST / "Saved/Linux/l169/user" / condition / run_name + zendata = HOST / "Saved/Linux/l169/zen" / condition / run_name + ddc = HOST / "Saved/Linux/l169/DDC" + trace = out / f"{run_name}.trace.jsonl" + log = out / f"{run_name}.log" + stdout = out / f"{run_name}.stdout.log" + for directory in (out, export, userdir, zendata, ddc): + directory.mkdir(parents=True, exist_ok=True) + trace.unlink(missing_ok=True) + spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M) + command = [ + str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", + "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", + "-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer", + "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", + f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", + "-archetype=TunnelNetwork", "-slot=0", "-modes=export", "-gameconfig=1", + "-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25", + "-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", + "-coreplantest=0", "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", + "-corehybridmesher=0", f"-out={export}", + "-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=3000.0", + f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}", + f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}", + "-voxel.TestExitSeconds=0.0", "-voxel.TestMoveDirection=1,0,0", + "-voxel.TestExitOnSteady=0", "-coreworldstreamtrace=1", + f"-voxel.StartupTraceSpawnCm={spawn_cm}", + "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", + "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", + f"-abslog={log}", + ] + if condition in ("early_override", "candidate"): + command += ["-voxel.StreamingFastTravelWaitForAllCriticalTiles=0", + "-voxel.StreamingFastTravelApplyBudgetMilliseconds=1.0"] + if tile_hash_dump is not None: + tile_hash_dump.parent.mkdir(parents=True, exist_ok=True) + tile_hash_dump.unlink(missing_ok=True) + command.append(f"-voxel.TileHashDump={tile_hash_dump.resolve()}") + started = time.monotonic() + with stdout.open("w", encoding="utf-8") as stream: + process = subprocess.run( + ["taskset", "-c", "0-11", "timeout", "1740s", *command], + cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, check=False, timeout=1740) + return summarize(condition, run_name, trace, log, process.returncode, + time.monotonic() - started, diagnostic_override) + + +def main() -> int: + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--diagnostic", action="store_true", + help="one early-admission trace on the selected diagnostic binary") + modes.add_argument("--matrix", action="store_true", + help="warm both binaries, then run seven interleaved pairs with only the early-admission switches overridden") + modes.add_argument("--geometry-hash-check", action="store_true", + help="compare shared tile hashes with defaults vs the promoted switches") + args = parser.parse_args() + missing = [p for p in (EDITOR, HOST / "VFHost.uproject", ROOT / "VoxelForge.uplugin") + if not p.exists()] + if missing: + raise SystemExit(f"missing UE runner input: {missing}") + OUT.mkdir(parents=True, exist_ok=True) + + if args.diagnostic: + warmup = run("early_override", "diagnostic-warmup", diagnostic_override=True) + row = run("early_override", "l169-default-base-diagnostic", diagnostic_override=True) + row["warmup"] = warmup + path = OUT / "diagnostic.json" + path.write_text(json.dumps(row, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("[L169Diagnostic] " + json.dumps(row, sort_keys=True), flush=True) + return 0 + + if args.geometry_hash_check: + hash_dir = OUT / "geometry-hash" + baseline_dump = hash_dir / "baseline.json" + candidate_dump = hash_dir / "candidate.json" + run("baseline", "geometry-hash", tile_hash_dump=baseline_dump) + run("candidate", "geometry-hash", tile_hash_dump=candidate_dump) + baseline_data = json.loads(baseline_dump.read_text(encoding="utf-8")) + candidate_data = json.loads(candidate_dump.read_text(encoding="utf-8")) + key_fields = ("level", "tile_x", "tile_y", "tile_z") + + def indexed(data: dict) -> dict[tuple[int, ...], tuple]: + return {tuple(int(record[field]) for field in key_fields): ( + str(record.get("geometry_hash", "")), + int(record.get("triangle_count", -1)), + bool(record.get("empty", False))) + for record in data.get("records", [])} + + base = indexed(baseline_data) + candidate = indexed(candidate_data) + common = base.keys() & candidate.keys() + mismatches = [key for key in common if base[key] != candidate[key]] + result = {"baseline_records": len(base), "candidate_records": len(candidate), + "common_records": len(common), "hash_mismatches": len(mismatches), + "baseline_only_records": len(base.keys() - candidate.keys()), + "candidate_only_records": len(candidate.keys() - base.keys()), + "baseline_dump": str(baseline_dump.relative_to(ROOT)), + "candidate_dump": str(candidate_dump.relative_to(ROOT))} + (hash_dir / "comparison.json").write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("[L169GeometryHash] " + json.dumps(result, sort_keys=True), flush=True) + return 0 if len(common) >= 100 and not mismatches else 1 + + warmups = [run("baseline", "warmup"), run("candidate", "warmup")] + rows = [] + partial = OUT / "runs-partial.json" + for pair, sequence in ORDER.items(): + for condition in sequence: + rows.append(run(condition, f"pair-{pair}")) + partial.write_text(json.dumps(rows, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + metrics = ("tick_p99_ms", "tick_worst_ms", "near_readiness_moving_mean_pct", + "forward_cone_readiness_moving_mean_pct", "far_holes_moving_median_pct", + "arrival_lod02_settled_s") + medians = {condition: {metric: statistics.median( + row[metric] for row in rows if row["condition"] == condition + and row.get(metric) is not None) for metric in metrics} + for condition in ("baseline", "candidate")} + by_pair = {pair: {row["condition"]: row for row in rows if row["run"] == f"pair-{pair}"} + for pair in ORDER} + deltas = [ + {"pair": pair, **{f"{metric}_delta": + by_pair[pair]["candidate"][metric] - by_pair[pair]["baseline"][metric] + for metric in metrics}} + for pair in ORDER + ] + summary = {"protocol": "Seven interleaved 30 m/s UE pairs using base defaults except for " + "the two early-admission switches at candidate values; 12 s move plus 5 s " + "dwell; eight effective tile slots; one warm-up per binary excluded.", + "defaults": {"incremental_desired_set": 1, + "streaming_fast_travel_wait_for_all_critical_tiles": 1, + "streaming_fast_travel_apply_budget_ms": 0.0, + "forward_first": 0, "rebuild_on_core_region_completion": 1, + "core_desired_cache_by_center": 0, + "incremental_desired_audit_every": 0, + "incremental_desired_materialization_audit_every": 0, + "incremental_desired_audit_inject_mismatch": 0}, + "candidate_switch_overrides": { + "streaming_fast_travel_wait_for_all_critical_tiles": 0, + "streaming_fast_travel_apply_budget_ms": 1.0}, + "warmups": warmups, "movement_medians": medians, + "paired_deltas_candidate_minus_baseline": deltas, "runs": rows} + (OUT / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + print("[L169Summary] " + json.dumps({"movement_medians": medians, + "paired_deltas": deltas}, sort_keys=True), flush=True) + all_runs_p99_pass = all(row["tick_p99_ms"] is not None and row["tick_p99_ms"] <= 11.0 + for row in rows) + all_runs_worst_pass = all(row["tick_worst_ms"] is not None + and row["tick_worst_ms"] <= 14.0 for row in rows) + candidate_worst = [row["tick_worst_ms"] for row in rows + if row["condition"] == "candidate"] + baseline_worst = [row["tick_worst_ms"] for row in rows + if row["condition"] == "baseline"] + gates = { + "every_run_p99_le_11_ms": all_runs_p99_pass, + "every_run_worst_le_14_ms": all_runs_worst_pass, + "candidate_worst_tick_median_not_worse": bool(candidate_worst and baseline_worst) + and statistics.median(candidate_worst) <= statistics.median(baseline_worst), + "near_readiness_up": medians["candidate"]["near_readiness_moving_mean_pct"] + > medians["baseline"]["near_readiness_moving_mean_pct"], + "forward_cone_readiness_up": medians["candidate"][ + "forward_cone_readiness_moving_mean_pct"] + > medians["baseline"]["forward_cone_readiness_moving_mean_pct"], + "far_holes_not_worse": medians["candidate"]["far_holes_moving_median_pct"] + <= medians["baseline"]["far_holes_moving_median_pct"], + "arrival_not_worse": medians["candidate"]["arrival_lod02_settled_s"] + <= medians["baseline"]["arrival_lod02_settled_s"], + } + summary["promotion_gates"] = gates + (OUT / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + print("[L169Gates] " + json.dumps(gates, sort_keys=True), flush=True) + return 0 if all(gates.values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 24803ab771b25c341bc8b365b52e2ff366549b7f Mon Sep 17 00:00:00 2001 From: Fr0zka Date: Mon, 2 Nov 2026 17:28:52 +0100 Subject: [PATCH 2/4] streaming: latch fast-travel early admission until the arrival set settles (L170, still opt-in) Switches verified inactive before first settlement; moving gains hold (near 43.7 -> 55.5%, far holes 1.8 -> 0%, p99 7.21 -> 6.48 ms, worst 12.32 -> 12.27 ms). Arrival median +1.28% (4/7 pairs slower) though the switches did no work then; defaults remain off (WaitForAllCriticalTiles=1, ApplyBudget=0). Source/: VoxelWorld.cpp, VoxelWorld.h. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1 --- Source/VoxelForge/Private/VoxelWorld.cpp | 17 +- Source/VoxelForge/Public/VoxelWorld.h | 2 + .../WorldAssemblerM3/Linux170/REPORT.md | 55 ++ .../Linux170/RunL170Movement.py | 490 ++++++++++++++++++ 4 files changed, 559 insertions(+), 5 deletions(-) create mode 100644 Standalone/Research/WorldAssemblerM3/Linux170/REPORT.md create mode 100644 Standalone/Research/WorldAssemblerM3/Linux170/RunL170Movement.py diff --git a/Source/VoxelForge/Private/VoxelWorld.cpp b/Source/VoxelForge/Private/VoxelWorld.cpp index 8bf7021..95f4533 100644 --- a/Source/VoxelForge/Private/VoxelWorld.cpp +++ b/Source/VoxelForge/Private/VoxelWorld.cpp @@ -9984,6 +9984,7 @@ void AVoxelWorld::ProcessPendingChunks(uint64 TickStartCycles) } } } + bInitialArrivalSetSettled = bInitialArrivalSetSettled || bInitialDesiredSetReady; const bool bStartupApplyOverridden = GVoxelForgeStreamingStartupApplyBoost <= 0 || GVoxelForgeStep1MaxMeshApplies > 0; if (bStartupApplyBoostStarted && !bStartupApplyBoostEnded @@ -10035,7 +10036,8 @@ void AVoxelWorld::ProcessPendingChunks(uint64 TickStartCycles) float ApplyBudgetMilliseconds = bUseStartupApplyBoost ? StreamingStartupApplyBudgetMilliseconds : (Settings ? FMath::Max(Settings->MaxMeshApplyMilliseconds, 0.0f) : 2.0f); - const bool bFastTravelApplyBudgetActive = bStreamingPlayerMoving && bSpeedAwareFastMode + const bool bFastTravelApplyBudgetActive = bInitialArrivalSetSettled + && bStreamingPlayerMoving && bSpeedAwareFastMode && GVoxelForgeStreamingFastTravelApplyBudgetMilliseconds > 0.0f; if (bFastTravelApplyBudgetActive) { @@ -13562,7 +13564,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw const int32 ConfiguredReadyQueueDepth = FMath::Clamp( GVoxelForgeStreamingReadyQueueDepth, 0, FMath::Min(64, FMath::Max(1, MaxTasks) * 4)); const bool bFastTravelAdmission = GVoxelForgeStreamingSchedulerPolicy == 3 - && bSpeedAwareFastMode; + && bInitialArrivalSetSettled && bSpeedAwareFastMode; const int32 ReadyQueueDepth = bFastTravelAdmission ? FMath::Max(ConfiguredReadyQueueDepth, 24) : ConfiguredReadyQueueDepth; const int32 TotalAdmissionCap = MaxTasks + ReadyQueueDepth; @@ -13695,7 +13697,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw ? AuthoredNearTaskReserve : ((bPawnMoving || !bWalkableRingReadyForReserve) ? QueueAwareNearTaskReserve : 0); const int32 NearTaskReserve = EffectiveNearTaskReserve; - const bool bPolicy3FastTravel = SchedulerPolicy == 3 && bSpeedAwareFastMode; + const bool bPolicy3FastTravel = SchedulerPolicy == 3 + && bInitialArrivalSetSettled && bSpeedAwareFastMode; // Fast travel keeps three quarters of the bounded ready queue for near detail and // leaves the final quarter open for horizon coverage. The walkable ring is submitted first. const int32 FastTravelNearCap = MaxTasks + (ReadyQueueDepth * 3) / 4; @@ -14999,6 +15002,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"desired_rebuilt\":%d," "\"submission_loop_ran\":%d,\"all_chunks_loaded\":%d," "\"walkable_ready\":%d,\"moving\":%d," + "\"initial_arrival_set_settled\":%d,\"fast_travel_mode\":%d," + "\"wait_for_all_critical_tiles\":%d," "\"core_region_compile_pending\":%d,\"core_ready\":%d,\"coarse_first\":%d," "\"dynamic_near_reserve\":%d,\"legacy_pending_result_cap\":%d," "\"core_region_concurrent_admission\":%d," @@ -15038,6 +15043,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw static_cast(GFrameCounter), CenterChunk.X, CenterChunk.Y, CenterChunk.Z, bDesiredRebuiltThisUpdate ? 1 : 0, bSubmissionLoopRan ? 1 : 0, bAllChunksLoaded ? 1 : 0, bWalkableRingReadyForReserve ? 1 : 0, bPawnMoving ? 1 : 0, + bInitialArrivalSetSettled ? 1 : 0, bPolicy3FastTravel ? 1 : 0, + GVoxelForgeStreamingFastTravelWaitForAllCriticalTiles, bCoreRegionCompilePending ? 1 : 0, IsVoxelForgeCoreReady() ? 1 : 0, GVoxelForgeStreamingCoarseFirst, GVoxelForgeStreamingDynamicNearReserve, GVoxelForgeStreamingLegacyPendingResultCap, @@ -15118,7 +15125,7 @@ void AVoxelWorld::LoadTile(const FVoxelTileKey& Tile, EVoxelTileWorkPriority Wor const int32 ConfiguredReadyQueueDepth = FMath::Clamp( GVoxelForgeStreamingReadyQueueDepth, 0, FMath::Min(64, FMath::Max(1, MaxTasks) * 4)); const bool bFastTravelAdmission = GVoxelForgeStreamingSchedulerPolicy == 3 - && bSpeedAwareFastMode; + && bInitialArrivalSetSettled && bSpeedAwareFastMode; const int32 ReadyQueueDepth = bFastTravelAdmission ? FMath::Max(ConfiguredReadyQueueDepth, 24) : ConfiguredReadyQueueDepth; const int32 TotalAdmissionCap = MaxTasks + ReadyQueueDepth; @@ -15520,7 +15527,7 @@ void AVoxelWorld::LoadCoreTile(const FVoxelTileKey& Tile, EVoxelTileWorkPriority const int32 ConfiguredReadyQueueDepth = FMath::Clamp( GVoxelForgeStreamingReadyQueueDepth, 0, FMath::Min(64, FMath::Max(1, MaxTasks) * 4)); const bool bFastTravelAdmission = GVoxelForgeStreamingSchedulerPolicy == 3 - && bSpeedAwareFastMode; + && bInitialArrivalSetSettled && bSpeedAwareFastMode; const int32 ReadyQueueDepth = bFastTravelAdmission ? FMath::Max(ConfiguredReadyQueueDepth, 24) : ConfiguredReadyQueueDepth; const int32 TotalAdmissionCap = MaxTasks + ReadyQueueDepth; diff --git a/Source/VoxelForge/Public/VoxelWorld.h b/Source/VoxelForge/Public/VoxelWorld.h index 9b8cbb1..0626c98 100644 --- a/Source/VoxelForge/Public/VoxelWorld.h +++ b/Source/VoxelForge/Public/VoxelWorld.h @@ -1247,6 +1247,8 @@ public: bool bSampleStacksStarted = false; bool bStartupApplyBoostStarted = false; bool bStartupApplyBoostEnded = false; + // Fast-travel admission stays disabled until the initial near LOD and collision set is ready. + bool bInitialArrivalSetSettled = false; uint64 StartupApplyBoostStartCycles = 0; // Trace the resolved startup budget once per world. / Tracer une fois le budget de demarrage resolu par monde. bool bStartupApplyConfigTraced = false; diff --git a/Standalone/Research/WorldAssemblerM3/Linux170/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux170/REPORT.md new file mode 100644 index 0000000..5f31e57 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux170/REPORT.md @@ -0,0 +1,55 @@ +# L170 — Gate early admission on settled arrival + +**Result:** The initial-arrival latch works: fast-travel admission and its 1 ms apply budget stayed inactive until the LOD0–2 desired set and required collision set settled, then became active during 30 m/s movement. Promotion still fails the arrival gate, so the source defaults remain off. The candidate was 1.282% slower on the per-condition arrival medians and was slower in four of seven pairs. + +## First-arrival census + +The UE matrix used seven interleaved W8 pairs against `assembler/world` `ca0bf76`, 30 m/s, 12 seconds of movement plus 5 seconds of dwell, and one excluded warm-up per binary. The candidate binary was compiled with the two early-admission defaults on; there were no CVar command-line overrides. Its raw first-settled LOD0–2 median was 1.691450 s versus 1.670039 s for baseline (+21.411 ms, +1.282%). The median paired delta was +12.176 ms; the candidate was not slower in only three pairs. Times from the first streaming sample to settlement had medians 1.2965 s and 1.3132 s respectively. + +I counted startup-window trace timers from each run's first streaming sample through its first settled LOD0–2 sample. Candidate fast-travel mode was active in zero pre-settlement scheduler samples, and its apply budget remained at the normal 2 ms. During that same window: + +- Level candidate scanning summed to 2.896 ms baseline and 3.758 ms candidate (paired median delta +0.895 ms). Partial level sorting fell from 2.195 ms to 0.727 ms (−1.452 ms paired median). +- Desired rebuild totals were effectively unchanged: 3.046 ms versus 3.047 ms per window (paired median delta −0.035 ms). The candidate spent 0.149 ms in its transition-index phase; the baseline has no matching phase. +- Result-drain time was 127.986 ms versus 131.521 ms (paired median delta +5.001 ms). Apply time was 121.509 ms versus 125.095 ms (+5.416 ms paired median), while the candidate had applied 10 fewer meshes by settlement at the paired median. + +These timers locate the largest measured difference in result draining and apply work, but they do not isolate why that work costs more; some phase timers may overlap and should not be added. The early-admission switches themselves did no work before settlement, so the census does not attribute the remaining arrival regression to those switches. + +## Movement results + +Medians over the seven 30 m/s moving windows: + +| Metric | `ca0bf76` | Candidate | Change | +|---|---:|---:|---:| +| Near LOD0–2 readiness | 43.65% | 55.46% | +11.82 pp | +| Forward-cone readiness | 15.79% | 18.69% | +2.90 pp | +| Far holes | 1.80% | 0.00% | −1.80 pp | +| Tick p99 | 7.214 ms | 6.476 ms | lower | +| Worst tick | 12.315 ms | 12.272 ms | lower | + +All runs met p99 ≤ 11 ms and worst tick ≤ 14 ms; the largest observed values across both conditions were 7.281 ms and 12.881 ms. Candidate fast-travel mode and the 1 ms budget were active in every moving run. The movement gains met the L169-relative gates; only the two arrival gates failed. + +## Defaults and checks + +| CVar | Final source default | Candidate test default | +|---|---:|---:| +| `voxel.StreamingFastTravelWaitForAllCriticalTiles` | 1 | 0 | +| `voxel.StreamingFastTravelApplyBudgetMilliseconds` | 0 ms | 1 ms | +| `voxel.StreamingIncrementalDesiredSet` | 1 | 1 | +| `voxel.StreamingFastTravelForwardFirst` | 0 | 0 | +| `voxel.StreamingRebuildOnCoreRegionCompletion` | 1 | 1 | +| `voxel.StreamingCoreDesiredCacheByCenter` | 0 | 0 | +| `voxel.StreamingIncrementalDesiredAuditEvery` | 0 | 0 | +| `voxel.StreamingIncrementalDesiredMaterializationAuditEvery` | 0 | 0 | +| `voxel.StreamingIncrementalDesiredAuditInjectMismatch` | 0 | 0 | + +- VFHost UE Linux Development builds passed for the candidate-on measurement binary and final defaults-off source (`-NoUBA -MaxParallelActions=4`); three existing deprecated RealtimeMesh API warnings remain. +- `Standalone/Tools/verify/verify.py fast` passed on the final source in 280.7 s: 268 passes, zero failures, five non-gating hash-baseline diagnostics. Worker-count determinism checks passed. +- The L170 movement runner completed all seven pairs. Its exit was nonzero only because the two arrival promotion gates failed. Runner AST parsing and `git diff --check` passed. +- UE W12 movement timing was not run; the VFHost procedure reported eight effective tile slots. Fast-tier W1/W8/W12 determinism checks ran. + +No geometry or evaluator code changed. Remaining risk is the unexplained arrival regression in result-drain/apply work; promotion should wait for a measurement that isolates that cost. + +Changed Unreal `Source/` files: + +- `Source/VoxelForge/Private/VoxelWorld.cpp` +- `Source/VoxelForge/Public/VoxelWorld.h` diff --git a/Standalone/Research/WorldAssemblerM3/Linux170/RunL170Movement.py b/Standalone/Research/WorldAssemblerM3/Linux170/RunL170Movement.py new file mode 100644 index 0000000..29dc7f8 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux170/RunL170Movement.py @@ -0,0 +1,490 @@ +#!/usr/bin/env python3 +"""L170 compiled-default UE movement comparison and startup-gate trace.""" +from __future__ import annotations + +import argparse +import json +import math +import shutil +import statistics +import subprocess +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[4] +HOST = Path("/home/jahni/VFHost") +EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd") +HOST_PLUGIN = HOST / "Binaries/Linux/libUnrealEditor-VoxelForge.so" +OUT = ROOT / "Saved/Linux/l170/ue/movement" +BIN_DIR = ROOT / "Saved/Linux/l170/bin" +SPAWN_M = (19.166015625, -14.2001953125, -753.3740234375) +MOVE_START_S = 6.0 +MOVE_DURATION_S = 12.0 + +BASELINE_BINARY = ROOT / "Saved/Linux/l169/bin/ca0bf76/libUnrealEditor-VoxelForge.so" +CANDIDATE_BINARY = BIN_DIR / "candidate-default-on/libUnrealEditor-VoxelForge.so" +DIAGNOSTIC_BINARY = CANDIDATE_BINARY + +EXPECTED = { + "baseline": {"wait": 1, "apply_budget_ms": 0.0, "incremental": 1}, + "candidate": {"wait": 0, "apply_budget_ms": 1.0, "incremental": 1}, +} + +ORDER = { + 1: ("baseline", "candidate"), + 2: ("candidate", "baseline"), + 3: ("baseline", "candidate"), + 4: ("candidate", "baseline"), + 5: ("baseline", "candidate"), + 6: ("candidate", "baseline"), + 7: ("baseline", "candidate"), +} + + +def percentile(values: list[float], fraction: float = 0.99) -> float | None: + if not values: + return None + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1)) + return ordered[index] + + +def percent(numerator: int | float, denominator: int | float) -> float | None: + return 100.0 * numerator / denominator if denominator > 0 else None + + +def read_events(path: Path) -> list[dict]: + rows = [] + if path.is_file(): + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not line.startswith("{"): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def summarize(condition: str, run_name: str, trace: Path, log: Path, exit_code: int, + wall_s: float, audit_or_diag: bool = False) -> dict: + expected = EXPECTED[condition] + events = read_events(trace) + backend = next((e for e in events if e.get("type") == "worker_backend_config"), {}) + scheduler_all = [e for e in events if e.get("type") == "scheduler_admission"] + if not scheduler_all: + raise RuntimeError(f"missing scheduler readback: {trace}") + if any(int(e.get("incremental_desired_set", -1)) != expected["incremental"] + for e in scheduler_all): + raise RuntimeError(f"incremental desired-set readback mismatch: {trace}") + if any(int(e.get("forward_first", -1)) != 0 + or int(e.get("rebuild_on_core_region_completion", -1)) != 1 + or int(e.get("core_desired_cache_by_center", -1)) != 0 + or int(e.get("incremental_desired_audit_every", -1)) != 0 + or int(e.get("incremental_desired_materialization_audit_every", -1)) != 0 + or int(e.get("incremental_desired_audit_inject_mismatch", -1)) != 0 + for e in scheduler_all): + raise RuntimeError(f"non-default L160/audit CVar readback: {trace}") + if (int(backend.get("streaming_fast_travel_wait_for_all_critical_tiles", -1)) + != expected["wait"] + or abs(float(backend.get("streaming_fast_travel_apply_budget_ms", -999.0)) + - expected["apply_budget_ms"]) > 0.01 + or int(backend.get("effective_tile_slots", -1)) != 8): + raise RuntimeError(f"early-admission/worker readback mismatch: {trace}; {backend}") + if condition == "candidate" and any( + int(e.get("wait_for_all_critical_tiles", -1)) != expected["wait"] + or "initial_arrival_set_settled" not in e + or "fast_travel_mode" not in e + for e in scheduler_all): + raise RuntimeError(f"missing L170 arrival-gate readback: {trace}") + + move_stop = next((e for e in events if e.get("type") == "streaming_test_move_stopped"), None) + if move_stop is None: + raise RuntimeError(f"missing move-stop event: {trace}") + stop_s = float(move_stop.get("t_s", 0.0)) + samples = [e for e in events if e.get("type") == "streaming_sample"] + moving = [e for e in samples if MOVE_START_S <= float(e.get("t_s", 0.0)) < stop_s + and float(e.get("speed_cm_s", 0.0)) >= 50.0] + dwell = [e for e in samples if float(e.get("t_s", 0.0)) >= stop_s + and float(e.get("speed_cm_s", 0.0)) < 50.0] + if exit_code != 0 or len(moving) < 40 or len(dwell) < 16: + raise RuntimeError(f"incomplete {run_name}: exit={exit_code}, move={len(moving)}, " + f"dwell={len(dwell)}") + move_start = min(float(e.get("t_s", 0.0)) for e in moving) + + work = [e for e in events if e.get("type") == "game_thread_work_frame" + and move_start <= float(e.get("t_s", 0.0)) < stop_s] + scheduler_moving = [e for e in scheduler_all + if move_start <= float(e.get("t_s", 0.0)) < stop_s] + rebuild_events = [e for e in events if e.get("type") == "desired_rebuild_breakdown" + and move_start <= float(e.get("t_s", 0.0)) < stop_s] + applies = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "apply_frame" + and move_start <= float(e.get("t_s", 0.0)) < stop_s} + scheduler = {int(e.get("frame", -1)): e for e in scheduler_all + if move_start <= float(e.get("t_s", 0.0)) < stop_s} + desired = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "desired_rebuild_breakdown" + and move_start <= float(e.get("t_s", 0.0)) < stop_s} + moving_tiles = [e for e in events if e.get("type") == "tile" + and move_start <= float(e.get("t_s", 0.0)) < stop_s] + apply_deferrals = [e for e in events if e.get("type") == "tile_apply_deferred" + and move_start <= float(e.get("t_s", 0.0)) < stop_s] + + frames = [] + for event in work: + frame = int(event.get("frame", -1)) + apply = applies.get(frame, {}) + admit = scheduler.get(frame, {}) + rebuild = desired.get(frame, {}) + frames.append({ + "frame": frame, + "tick_ms": float(event.get("tick_ms", 0.0)), + "update_chunks_ms": float(event.get("update_chunks_ms", 0.0)), + "pending_drain_ms": float(event.get("pending_drain_ms", 0.0)), + "apply_ms": float(apply.get("apply_ms", 0.0)), + "apply_budget_ms": float(apply.get("apply_budget_ms", 0.0)), + "meshes_applied": int(apply.get("meshes_applied", 0)), + "time_budget_hit": int(apply.get("time_limit_reached", 0)), + "desired_rebuild_ms": float(admit.get("desired_rebuild_ms", 0.0)), + "desired_rebuild_event_ms": float(rebuild.get("total_ms", 0.0)), + "overlay_materialize_ms": float(rebuild.get("overlay_materialize_ms", 0.0)), + "core_prepare_ms": float(rebuild.get("core_prepare_ms", 0.0)), + "core_delta_ms": float(rebuild.get("core_delta_ms", 0.0)), + "priority_sort_ms": float(rebuild.get("priority_sort_ms", 0.0)), + "critical_order_ms": float(rebuild.get("critical_order_ms", 0.0)), + "collision_stamp_ms": float(rebuild.get("collision_stamp_ms", 0.0)), + "transition_index_ms": float(rebuild.get("transition_index_ms", 0.0)), + "retained_region_ms": float(rebuild.get("retained_region_ms", 0.0)), + "region_reconcile_ms": float(rebuild.get("region_reconcile_ms", 0.0)), + "level_sort_ms": float(admit.get("level_sort_ms", 0.0)), + "level_candidate_scan_ms": float(admit.get("level_candidate_scan_ms", 0.0)), + "submit_tiles_ms": float(admit.get("submit_tiles_ms", 0.0)), + }) + + near = [] + cone = [] + far_holes = [] + for event in moving: + desired_by_lod = event.get("desired_by_lod", [0] * 9) + resident_by_lod = event.get("resident_by_lod", [0] * 9) + covered_by_lod = event.get("covered_by_lod", resident_by_lod) + near.append(percent(sum(resident_by_lod[:3]), sum(desired_by_lod[:3])) or 0.0) + cone.append(percent(int(event.get("near_forward_cone_resident", 0)), + int(event.get("near_forward_cone_desired", 0))) or 0.0) + far_holes.append(percent(sum(desired_by_lod[3:]) - sum(covered_by_lod[3:]), + sum(desired_by_lod[3:])) or 0.0) + settled = next((e for e in samples if float(e.get("t_s", 0.0)) < move_start + and float(e.get("speed_cm_s", 0.0)) < 50.0 + and sum(e.get("settled_by_lod", [0] * 9)[:3]) + >= sum(e.get("desired_by_lod", [0] * 9)[:3]) > 0), None) + if settled is None: + raise RuntimeError(f"missing initial LOD0-2 settled sample: {trace}") + arrival_settle_s = float(settled["t_s"]) + arrival_scheduler = [e for e in scheduler_all + if float(e.get("t_s", 0.0)) <= arrival_settle_s] + apply_all = [e for e in events if e.get("type") == "apply_frame"] + if condition == "candidate": + if any(int(e.get("fast_travel_mode", 0)) != 0 for e in arrival_scheduler): + raise RuntimeError(f"fast-travel admission activated before initial arrival settled: {trace}") + if any(float(e.get("apply_budget_ms", 0.0)) < 1.99 + and float(e.get("t_s", 0.0)) <= arrival_settle_s for e in apply_all): + raise RuntimeError(f"fast-travel apply cap activated before initial arrival settled: {trace}") + if not any(int(e.get("initial_arrival_set_settled", 0)) == 1 + and int(e.get("fast_travel_mode", 0)) == 1 + for e in scheduler_moving): + raise RuntimeError(f"fast-travel mode never activated after initial arrival settled: {trace}") + if not any(float(e.get("apply_budget_ms", 0.0)) <= 1.001 + for e in apply_all + if move_start <= float(e.get("t_s", 0.0)) < stop_s): + raise RuntimeError(f"compiled 1 ms apply cap never activated while moving: {trace}") + p99 = percentile([r["tick_ms"] for r in frames]) + worst = max(frames, key=lambda r: r["tick_ms"], default={}) + row = { + "condition": condition, + "run": run_name, + "diagnostic_override": audit_or_diag, + "source_default_incremental": expected["incremental"], + "early_admission_wait_for_critical": expected["wait"], + "early_admission_apply_budget_ms": expected["apply_budget_ms"], + "speed_mps": 30, + "effective_tile_slots": backend.get("effective_tile_slots"), + "moving_samples": len(moving), + "dwell_samples": len(dwell), + "tick_p99_ms": p99, + "tick_worst_ms": worst.get("tick_ms"), + "worst_frame": worst, + "near_readiness_moving_mean_pct": statistics.mean(near) if near else None, + "forward_cone_readiness_moving_mean_pct": statistics.mean(cone) if cone else None, + "far_holes_moving_median_pct": statistics.median(far_holes) if far_holes else None, + "arrival_lod02_settled_s": float(settled.get("t_s", 0.0)) if settled else None, + "arrival_fast_travel_modes_before_settle": sum( + int(e.get("fast_travel_mode", 0)) != 0 for e in arrival_scheduler), + "arrival_gate_was_settled_before_move": any( + int(e.get("initial_arrival_set_settled", 0)) != 0 + for e in scheduler_all if float(e.get("t_s", 0.0)) < move_start), + "moving_fast_travel_mode_frames": sum( + int(e.get("fast_travel_mode", 0)) != 0 for e in scheduler_moving), + "moving_one_ms_apply_budget_frames": sum( + float(e.get("apply_budget_ms", 0.0)) <= 1.001 + for e in apply_all if move_start <= float(e.get("t_s", 0.0)) < stop_s), + "desired_rebuild_count": len(rebuild_events), + "incremental_materialization_updates": sum( + int(e.get("materialization_update", 0)) != 0 for e in events + if e.get("type") == "desired_rebuild_breakdown" + and move_start <= float(e.get("t_s", 0.0)) < stop_s), + "core_incremental_deltas": sum( + int(e.get("incremental_delta", 0)) != 0 for e in events + if e.get("type") == "core_desired_cache" + and move_start <= float(e.get("t_s", 0.0)) < stop_s), + "level_sort_samples_over_1ms": sum( + float(e.get("level_sort_ms", 0.0)) > 1.0 for e in scheduler_moving), + "level_sort_max_ms": max((float(e.get("level_sort_ms", 0.0)) + for e in scheduler_moving), default=0.0), + "desired_rebuild_events_over_5ms": sum( + float(e.get("total_ms", 0.0)) > 5.0 for e in rebuild_events), + "desired_rebuild_max_ms": max( + (float(e.get("total_ms", 0.0)) for e in rebuild_events), default=0.0), + "overlay_materialize_events_over_2ms": sum( + float(e.get("overlay_materialize_ms", 0.0)) > 2.0 for e in rebuild_events), + "overlay_materialize_max_ms": max( + (float(e.get("overlay_materialize_ms", 0.0)) for e in rebuild_events), default=0.0), + "desired_subphase_max_ms": { + key: max((float(e.get(key, 0.0)) for e in rebuild_events), default=0.0) + for key in ("core_prepare_ms", "overlay_materialize_ms", "priority_sort_ms", + "critical_order_ms", "collision_stamp_ms", "transition_index_ms", + "retained_region_ms", "region_reconcile_ms")}, + "apply_budget_hit_frames": sum(f["time_budget_hit"] for f in frames), + "apply_frame_count": sum(f["meshes_applied"] > 0 for f in frames), + "apply_ms_max": max((f["apply_ms"] for f in frames), default=0.0), + "tiles_over_1ms_apply": sum(float(e.get("apply_s", 0.0)) > 0.001 + for e in moving_tiles), + "apply_preflight_deferred_count": len(apply_deferrals), + "apply_preflight_deferred_max_estimate_ms": max( + (float(e.get("estimated_apply_ms", 0.0)) for e in apply_deferrals), default=0.0), + "apply_preflight_deferred_max_triangles": max( + (int(e.get("triangles", 0)) for e in apply_deferrals), default=0), + "largest_tile_applies": sorted(({ + "apply_ms": float(e.get("apply_s", 0.0)) * 1000.0, + "triangles": int(e.get("triangles", 0)), + "vertices": int(e.get("vertices", 0)), + "level": int(e.get("level", -1)), + } for e in moving_tiles), + key=lambda e: e["apply_ms"], reverse=True)[:8], + "top_ticks": sorted(frames, key=lambda r: r["tick_ms"], reverse=True)[:8], + "trace": str(trace.relative_to(ROOT)), + "log": str(log.relative_to(ROOT)), + "wall_s": round(wall_s, 3), + } + return row + + +def run(condition: str, run_name: str, diagnostic_override: bool = False, + tile_hash_dump: Path | None = None) -> dict: + if condition == "baseline": + binary = BASELINE_BINARY + elif condition == "candidate": + binary = CANDIDATE_BINARY + else: + raise ValueError(condition) + if not binary.is_file(): + raise FileNotFoundError(binary) + HOST_PLUGIN.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(binary, HOST_PLUGIN) + out = OUT / condition + export = HOST / "Plugins/VoxelForge/Saved/Linux/l170" / condition / run_name + userdir = HOST / "Saved/Linux/l170/user" / condition / run_name + zendata = HOST / "Saved/Linux/l170/zen" / condition / run_name + ddc = HOST / "Saved/Linux/l170/DDC" + trace = out / f"{run_name}.trace.jsonl" + log = out / f"{run_name}.log" + stdout = out / f"{run_name}.stdout.log" + for directory in (out, export, userdir, zendata, ddc): + directory.mkdir(parents=True, exist_ok=True) + trace.unlink(missing_ok=True) + spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M) + command = [ + str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", + "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", + "-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer", + "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", + f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", + "-archetype=TunnelNetwork", "-slot=0", "-modes=export", "-gameconfig=1", + "-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25", + "-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", + "-coreplantest=0", "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", + "-corehybridmesher=0", f"-out={export}", + "-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=3000.0", + f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}", + f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}", + "-voxel.TestExitSeconds=0.0", "-voxel.TestMoveDirection=1,0,0", + "-voxel.TestExitOnSteady=0", "-coreworldstreamtrace=1", + f"-voxel.StartupTraceSpawnCm={spawn_cm}", + "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", + "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", + f"-abslog={log}", + ] + if tile_hash_dump is not None: + tile_hash_dump.parent.mkdir(parents=True, exist_ok=True) + tile_hash_dump.unlink(missing_ok=True) + command.append(f"-voxel.TileHashDump={tile_hash_dump.resolve()}") + started = time.monotonic() + with stdout.open("w", encoding="utf-8") as stream: + process = subprocess.run( + ["taskset", "-c", "0-11", "timeout", "1740s", *command], + cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, check=False, timeout=1740) + return summarize(condition, run_name, trace, log, process.returncode, + time.monotonic() - started, diagnostic_override) + + +def main() -> int: + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--diagnostic", action="store_true", + help="one candidate trace with compiled defaults and no command-line switch overrides") + modes.add_argument("--matrix", action="store_true", + help="warm both binaries, then run seven interleaved pairs with compiled defaults and no switch overrides") + modes.add_argument("--geometry-hash-check", action="store_true", + help="compare shared tile hashes from the compiled-default binaries") + args = parser.parse_args() + missing = [p for p in (EDITOR, HOST / "VFHost.uproject", ROOT / "VoxelForge.uplugin") + if not p.exists()] + if missing: + raise SystemExit(f"missing UE runner input: {missing}") + OUT.mkdir(parents=True, exist_ok=True) + + if args.diagnostic: + warmup = run("candidate", "diagnostic-warmup", diagnostic_override=True) + row = run("candidate", "l170-default-diagnostic", diagnostic_override=True) + row["warmup"] = warmup + path = OUT / "diagnostic.json" + path.write_text(json.dumps(row, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("[L169Diagnostic] " + json.dumps(row, sort_keys=True), flush=True) + return 0 + + if args.geometry_hash_check: + hash_dir = OUT / "geometry-hash" + baseline_dump = hash_dir / "baseline.json" + candidate_dump = hash_dir / "candidate.json" + run("baseline", "geometry-hash", tile_hash_dump=baseline_dump) + run("candidate", "geometry-hash", tile_hash_dump=candidate_dump) + baseline_data = json.loads(baseline_dump.read_text(encoding="utf-8")) + candidate_data = json.loads(candidate_dump.read_text(encoding="utf-8")) + key_fields = ("level", "tile_x", "tile_y", "tile_z") + + def indexed(data: dict) -> dict[tuple[int, ...], tuple]: + return {tuple(int(record[field]) for field in key_fields): ( + str(record.get("geometry_hash", "")), + int(record.get("triangle_count", -1)), + bool(record.get("empty", False))) + for record in data.get("records", [])} + + base = indexed(baseline_data) + candidate = indexed(candidate_data) + common = base.keys() & candidate.keys() + mismatches = [key for key in common if base[key] != candidate[key]] + result = {"baseline_records": len(base), "candidate_records": len(candidate), + "common_records": len(common), "hash_mismatches": len(mismatches), + "baseline_only_records": len(base.keys() - candidate.keys()), + "candidate_only_records": len(candidate.keys() - base.keys()), + "baseline_dump": str(baseline_dump.relative_to(ROOT)), + "candidate_dump": str(candidate_dump.relative_to(ROOT))} + (hash_dir / "comparison.json").write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("[L169GeometryHash] " + json.dumps(result, sort_keys=True), flush=True) + return 0 if len(common) >= 100 and not mismatches else 1 + + warmups = [run("baseline", "warmup"), run("candidate", "warmup")] + rows = [] + partial = OUT / "runs-partial.json" + for pair, sequence in ORDER.items(): + for condition in sequence: + rows.append(run(condition, f"pair-{pair}")) + partial.write_text(json.dumps(rows, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + metrics = ("tick_p99_ms", "tick_worst_ms", "near_readiness_moving_mean_pct", + "forward_cone_readiness_moving_mean_pct", "far_holes_moving_median_pct", + "arrival_lod02_settled_s") + medians = {condition: {metric: statistics.median( + row[metric] for row in rows if row["condition"] == condition + and row.get(metric) is not None) for metric in metrics} + for condition in ("baseline", "candidate")} + by_pair = {pair: {row["condition"]: row for row in rows if row["run"] == f"pair-{pair}"} + for pair in ORDER} + deltas = [ + {"pair": pair, **{f"{metric}_delta": + by_pair[pair]["candidate"][metric] - by_pair[pair]["baseline"][metric] + for metric in metrics}} + for pair in ORDER + ] + summary = {"protocol": "Seven interleaved 30 m/s UE pairs using compiled defaults; " + "candidate defaults enable the two arrival-gated fast-travel switches; no " + "command-line switch overrides; 12 s move plus 5 s dwell; eight effective " + "tile slots; one warm-up per binary excluded.", + "defaults": {"incremental_desired_set": 1, + "streaming_fast_travel_wait_for_all_critical_tiles": 0, + "streaming_fast_travel_apply_budget_ms": 1.0, + "forward_first": 0, "rebuild_on_core_region_completion": 1, + "core_desired_cache_by_center": 0, + "incremental_desired_audit_every": 0, + "incremental_desired_materialization_audit_every": 0, + "incremental_desired_audit_inject_mismatch": 0}, + "warmups": warmups, "movement_medians": medians, + "paired_deltas_candidate_minus_baseline": deltas, "runs": rows} + (OUT / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + print("[L170Summary] " + json.dumps({"movement_medians": medians, + "paired_deltas": deltas}, sort_keys=True), flush=True) + all_runs_p99_pass = all(row["tick_p99_ms"] is not None and row["tick_p99_ms"] <= 11.0 + for row in rows) + all_runs_worst_pass = all(row["tick_worst_ms"] is not None + and row["tick_worst_ms"] <= 14.0 for row in rows) + candidate_worst = [row["tick_worst_ms"] for row in rows + if row["condition"] == "candidate"] + baseline_worst = [row["tick_worst_ms"] for row in rows + if row["condition"] == "baseline"] + arrival_pairs_not_slower = sum( + by_pair[pair]["candidate"]["arrival_lod02_settled_s"] + <= by_pair[pair]["baseline"]["arrival_lod02_settled_s"] + for pair in ORDER) + arrival_median_delta_pct = 100.0 * ( + medians["candidate"]["arrival_lod02_settled_s"] + / medians["baseline"]["arrival_lod02_settled_s"] - 1.0) + near_gain_pp = (medians["candidate"]["near_readiness_moving_mean_pct"] + - medians["baseline"]["near_readiness_moving_mean_pct"]) + forward_gain_pp = (medians["candidate"]["forward_cone_readiness_moving_mean_pct"] + - medians["baseline"]["forward_cone_readiness_moving_mean_pct"]) + far_hole_reduction_pp = (medians["baseline"]["far_holes_moving_median_pct"] + - medians["candidate"]["far_holes_moving_median_pct"]) + gate_metrics = { + "arrival_delta_pct": arrival_median_delta_pct, + "arrival_pairs_not_slower": arrival_pairs_not_slower, + "near_gain_pp": near_gain_pp, + "forward_gain_pp": forward_gain_pp, + "far_hole_reduction_pp": far_hole_reduction_pp, + } + gates = { + "every_run_p99_le_11_ms": all_runs_p99_pass, + "every_run_worst_le_14_ms": all_runs_worst_pass, + "candidate_p99_median_not_worse": medians["candidate"]["tick_p99_ms"] + <= medians["baseline"]["tick_p99_ms"], + "candidate_worst_tick_median_not_worse": bool(candidate_worst and baseline_worst) + and statistics.median(candidate_worst) <= statistics.median(baseline_worst), + "near_gain_at_least_75pct_of_l169": near_gain_pp >= 9.63, + "forward_gain_at_least_75pct_of_l169": forward_gain_pp >= 1.81, + "far_hole_reduction_at_least_75pct_of_l169": far_hole_reduction_pp >= 1.35, + "arrival_median_within_0_5pct": abs(arrival_median_delta_pct) <= 0.5, + "arrival_not_slower_in_majority_of_pairs": arrival_pairs_not_slower >= 4, + } + summary["promotion_gates"] = gates + summary["promotion_gate_metrics"] = gate_metrics + (OUT / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + print("[L170Gates] " + json.dumps({"checks": gates, "metrics": gate_metrics}, + sort_keys=True), flush=True) + return 0 if all(gates.values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 9d088ef96c158bb4f3927d2a8423f6e75e2ba5c9 Mon Sep 17 00:00:00 2001 From: Fr0zka Date: Mon, 2 Nov 2026 18:08:43 +0100 Subject: [PATCH 3/4] research: UE arrival A/A could not run (L171) Two identical ca0bf76 VFHost builds staged together crash at BeginPlay after snapshot parsing (surface_strates=858993460 = 0x33333333, std::length_error). No arrival pairs measured. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1 --- Standalone/Research/WorldAssemblerM3/Linux171/REPORT.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Standalone/Research/WorldAssemblerM3/Linux171/REPORT.md diff --git a/Standalone/Research/WorldAssemblerM3/Linux171/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux171/REPORT.md new file mode 100644 index 0000000..5142c27 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux171/REPORT.md @@ -0,0 +1,9 @@ +# L171 — UE arrival measurement + +**Result:** Not measured. The A/A warm-up aborts before the first streaming sample, so this round cannot say whether L170’s +1.28% arrival difference exceeds UE noise or identify its cause. + +Two `assembler/world` `ca0bf76` VFHost Development builds completed and produced identical VoxelForge, VoxelForgeCore, and VoxelForgeEditor binaries. With that matched module set staged, the A/A warm-up reaches `world_assembly_plan` with one surface stratum, then logs `surface_strates=858993460` after snapshot parsing and terminates with `std::length_error: vector` during `AVoxelWorld::BeginPlay`. The trace has no scheduler readback or streaming sample. This points to a Core snapshot/module compatibility failure; the exact cause is unproven. + +Valid arrival pairs: A/A **0/15**, switches-on **0/15**, switches-off **0/15**. No medians, paired spread, or slower counts are available. The A/B batches were not started because the A/A control failed validation. Failed launches are logged under `Saved/Linux/l171/ue/arrival/`. + +The baseline and L170 HEAD-off VFHost builds passed. The runner compiled and its CLI loaded; `git diff --check` passed. No Unreal `Source/` files changed. The full arrival matrix and fast verification did not run: the UE control never reached measurement. From 52cb3d7a238aba599ee921d64713ae28e1a340f3 Mon Sep 17 00:00:00 2001 From: Fr0zka Date: Mon, 2 Nov 2026 20:27:24 +0100 Subject: [PATCH 4/4] streaming: restore same-level refill; early admission on by default; fixed UE harness (L172) Scheduler stable-sorts the whole level and checks capacity per admission (refill when slots free mid-pass); regression test with the old partial-sort path as negative control. Harness repoints Source/Standalone/uplugin together, records root HEAD/status, builds baselines from fresh worktrees, gates on DWARF struct sizes (480/480/480). Frame-resolution arrival: A/A p90 36.8 ms; fixed-on vs fixed-off -12.2 ms median (4/10 slower). 30 m/s: near 43.3 -> 55.3%, far holes 1.8 -> 0%, p99 <= 7.46 ms, worst <= 12.3 ms. Defaults: StreamingFastTravelWaitForAllCriticalTiles=0, StreamingFastTravelApplyBudgetMilliseconds=1.0, StreamingIncrementalDesiredSet=1, CoreRegionConcurrentAdmission=1, StreamingFastTravelForwardFirst=0, StreamingRebuildOnCoreRegionCompletion=1, StreamingCoreDesiredCacheByCenter=0, audits 0. Source/: VoxelWorld.cpp, VoxelStreamingAdmissionOrder.h (new). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1 --- .../Private/VoxelStreamingAdmissionOrder.h | 28 + Source/VoxelForge/Private/VoxelWorld.cpp | 44 +- .../WorldAssemblerM3/Linux172/REPORT.md | 62 ++ .../Linux172/RunL172Movement.py | 868 ++++++++++++++++++ .../RunStreamingAdmissionOrderTest.py | 59 ++ .../Linux172/TestStreamingAdmissionOrder.cpp | 97 ++ Standalone/Tools/verify/verify.py | 15 + 7 files changed, 1149 insertions(+), 24 deletions(-) create mode 100644 Source/VoxelForge/Private/VoxelStreamingAdmissionOrder.h create mode 100644 Standalone/Research/WorldAssemblerM3/Linux172/REPORT.md create mode 100644 Standalone/Research/WorldAssemblerM3/Linux172/RunL172Movement.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux172/RunStreamingAdmissionOrderTest.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux172/TestStreamingAdmissionOrder.cpp diff --git a/Source/VoxelForge/Private/VoxelStreamingAdmissionOrder.h b/Source/VoxelForge/Private/VoxelStreamingAdmissionOrder.h new file mode 100644 index 0000000..54287f2 --- /dev/null +++ b/Source/VoxelForge/Private/VoxelStreamingAdmissionOrder.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace VoxelStreamingAdmissionOrder +{ + template + void StableSortCandidates(Iterator First, Iterator Last, Less Compare) + { + std::stable_sort(First, Last, Compare); + } + + template + void AdmitInOrder(const Range& Candidates, IsAtCapacity AtCapacity, + SubmitCandidate Submit, OnCapacityReached CapacityReached) + { + for (const auto& Candidate : Candidates) + { + if (AtCapacity()) + { + CapacityReached(); + break; + } + Submit(Candidate); + } + } +} diff --git a/Source/VoxelForge/Private/VoxelWorld.cpp b/Source/VoxelForge/Private/VoxelWorld.cpp index 95f4533..2245867 100644 --- a/Source/VoxelForge/Private/VoxelWorld.cpp +++ b/Source/VoxelForge/Private/VoxelWorld.cpp @@ -2,6 +2,7 @@ // Implementation of the voxel world manager #include "VoxelWorld.h" +#include "VoxelStreamingAdmissionOrder.h" #include "VoxelDiffLayer.h" #include "RealtimeMeshComponent.h" #include "RealtimeMeshSimple.h" @@ -262,7 +263,7 @@ namespace GVoxelForgeCoreRegionConcurrentAdmission, TEXT("Allow tiles independent of a pending Core region compile to proceed; 0 restores the global streaming stop.")); - int32 GVoxelForgeStreamingFastTravelWaitForAllCriticalTiles = 1; + int32 GVoxelForgeStreamingFastTravelWaitForAllCriticalTiles = 0; FAutoConsoleVariableRef CVarVoxelForgeStreamingFastTravelWaitForAllCriticalTiles( TEXT("voxel.StreamingFastTravelWaitForAllCriticalTiles"), GVoxelForgeStreamingFastTravelWaitForAllCriticalTiles, @@ -361,7 +362,7 @@ namespace TEXT("voxel.StreamingStartupApplyBoost"), GVoxelForgeStreamingStartupApplyBoost, TEXT("Bounded cold-view upload budget (default on): 1 allows a 14 ms / 24-mesh cap until the initial desired set and walkable ring are ready; 0 uses the authored per-frame budget.")); - float GVoxelForgeStreamingFastTravelApplyBudgetMilliseconds = 0.0f; + float GVoxelForgeStreamingFastTravelApplyBudgetMilliseconds = 1.0f; FAutoConsoleVariableRef CVarVoxelForgeStreamingFastTravelApplyBudgetMilliseconds( TEXT("voxel.StreamingFastTravelApplyBudgetMilliseconds"), GVoxelForgeStreamingFastTravelApplyBudgetMilliseconds, @@ -14265,17 +14266,14 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw { const uint64 TraceLevelSortStartCycles = bTraceStreamingScheduler ? FPlatformTime::Cycles64() : 0; - const int32 LevelSubmissionCount = FMath::Min( - LevelTiles.Num(), FMath::Max( - 0, AdmissionCap - GetGenerationAdmissionCount())); const bool bUseForwardFirstOrder = GVoxelForgeStreamingFastTravelForwardFirst != 0 && bPolicy3FastTravel && Level <= 2; - if (LevelSubmissionCount > 0 && bUseForwardFirstOrder) + if (bUseForwardFirstOrder) { SortNearTilesForwardFirst(LevelTiles); } - else if (LevelSubmissionCount > 0) + else { FVector Heading = GetActorTransform().InverseTransformVectorNoScale(PlayerHeading); if (Heading.IsNearlyZero() && IsValid(PlayerPawn)) @@ -14288,12 +14286,10 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw const double TileSizeCm = static_cast(VF_ClipLevelExtentVoxels(Level)) * GetActiveVoxelSizeCm(); const bool bUseFastNearLeadOrder = bPolicy3FastTravel && Level <= 2; - // The full order is deterministic because the comparator breaks every tie - // with the tile coordinates. Select only the prefix this admission pass can - // submit; sorting every outstanding key made a small worker cap pay O(N log N). - std::partial_sort(LevelTiles.GetData(), - LevelTiles.GetData() + LevelSubmissionCount, - LevelTiles.GetData() + LevelTiles.Num(), + // Keep the complete stable order. Capacity can open while this pass is + // submitting; truncating to the pass-start count starves this level's refill. + VoxelStreamingAdmissionOrder::StableSortCandidates( + LevelTiles.GetData(), LevelTiles.GetData() + LevelTiles.Num(), [&](const FVoxelTileKey& A, const FVoxelTileKey& B) { const FVector CenterA( @@ -14348,7 +14344,6 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw if (A.Coord.Y != B.Coord.Y) return A.Coord.Y < B.Coord.Y; return A.Coord.Z < B.Coord.Z; }); - LevelTiles.SetNum(LevelSubmissionCount, EAllowShrinking::No); } if (bTraceStreamingScheduler) { @@ -14358,18 +14353,19 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw } const uint64 TraceLevelAdmissionStartCycles = bTraceStreamingScheduler ? FPlatformTime::Cycles64() : 0; - for (const FVoxelTileKey& T : LevelTiles) - { - if (GetGenerationAdmissionCount() >= AdmissionCap) + VoxelStreamingAdmissionOrder::AdmitInOrder(LevelTiles, + [&]() { return GetGenerationAdmissionCount() >= AdmissionCap; }, + [&](const FVoxelTileKey& T) + { + if (PendingTiles.Contains(T)) return; + if (LoadedTiles.Contains(T)) return; + SubmitTileWithTraceCount(T, Priority); + ++Submitted; + }, + [&]() { if (Level >= 0 && Level < 9) { ++SchedulerLevelCapBreaks[Level]; } - break; - } - if (PendingTiles.Contains(T)) continue; - if (LoadedTiles.Contains(T)) continue; - SubmitTileWithTraceCount(T, Priority); - ++Submitted; - } + }); if (bTraceStreamingScheduler) { LevelAdmissionMilliseconds += FPlatformTime::ToSeconds64( diff --git a/Standalone/Research/WorldAssemblerM3/Linux172/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux172/REPORT.md new file mode 100644 index 0000000..ae3054e --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux172/REPORT.md @@ -0,0 +1,62 @@ +# L172 — repair the UE harness and restore level refill + +**Result:** The scheduler now keeps the full stable candidate order and refills a level when a slot opens during its pass. The UE harness uses all three links from one tree and rejects mixed DWARF layouts. The 10-pair decision passed, so early admission is now the default. The final default-on UE build and fast tier passed. + +## L171 findings and harness + +The reviewer’s mixed-build diagnosis matches the saved evidence. L171’s A/A warm-up logged `surface_strates=1`, then `858993460` (`0x33333334`) before throwing `std::length_error: vector`. Its `aa-build-1.log` has only two link actions and no `Compile VoxelWorld.cpp`; the staged `aa-build-1`, `aa-build-2`, and `l170-head-off` plugin binaries contain the L170-only `initial_arrival_set_settled` and `tile_apply_deferred` UTF-16 strings, while the saved `ca0bf76` binary does not. The L171 DWARF inspection showed `WorldAssemblyStratumPlanV1` as 600 bytes in the stale `VoxelWorld.cpp.o` and 480 bytes in the adapter and Core shim. This is consistent with the vector corruption being an ABI mismatch. + +At L172 start, `Source`, `Standalone`, and `VoxelForge.uplugin` in `VFHost/PluginDeps/VoxelForge` all resolved to the main checkout on `wip/l141-liminal`. Earlier round reports document restoring `Source` alone. L172 switches all three links together and asserts their common root. Each run manifest records that root’s `HEAD` and `git status --porcelain`. The `ca0bf76` comparison used a fresh detached `git worktree` with its own `Source` and `Standalone`; it built `VoxelWorld.cpp`, passed the 480/480/480-byte DWARF gate, and ran the owner world in its warm-up and all ten pairs. Fixed-tree builds also compiled `VoxelWorld.cpp` and passed the same gate. All three VFHost links were restored to the main checkout after the last build and run. + +## Scheduler change + +`VoxelWorld.cpp` no longer sorts only the pass-start free-slot prefix and truncates `LevelTiles`. It stable-sorts the complete level and checks current capacity as each candidate is admitted. This preserves full-sort order and lets the same level refill when work completes mid-pass. The regression test frees a slot after the first submission, checks the submitted sequence against the full-sort order, and includes the old partial-sort/truncation path as a negative control. + +The frame-resolution arrival is the first `scheduler_admission` frame with both `desired_missing_by_lod[0:3]` and `pending_by_lod[0:3]` summing to zero. Re-scoring saved L169 and L170 traces gives paired median deltas of **+48.762 ms** and **+60.475 ms** respectively; each candidate was slower in all 7 pairs (14/14 combined). The 0.25 s `streaming_sample` arrival had obscured this difference. + +## L172 movement results + +Runs used the same fixed binary for both CVar arms, 30 m/s, one warm-up per arm, and ten counterbalanced pairs per batch. The measured movement window was 12 s (first moving samples near 6.05 s, stop near 18.02 s). The trace continued through 45.00 s after movement; those post-move samples were excluded from movement metrics. The pre-run load was 0.45 / 1.43 / 2.00 (1/5/15 min), with no UE or build process running. The harness reported eight effective tile slots. + +| Check | Result | +| --- | ---: | +| A/A absolute paired-delta p90 | 36.815 ms | +| Fixed-off slower than true `ca0bf76` | 6/10 pairs (stop rule: 8/10) | +| Fixed-on minus fixed-off arrival, paired median | −12.203 ms | +| Fixed-on slower on arrival | 4/10 pairs | +| Arrival tolerance | 36.815 ms | +| Near readiness change, fixed-on vs fixed-off | +11.951 pp (55.276% vs 43.325%) | +| Near readiness change, fixed-on vs `ca0bf76` | +12.457 pp (55.276% vs 42.820%) | +| Far-hole change, fixed-on vs either control | −1.800 pp (0.000% vs 1.800%) | +| Fixed-on maximum p99 / worst tick | 7.460 / 12.305 ms | +| Maximum p99 / worst tick across both on/off arms | 7.460 / 13.266 ms | + +All decision gates pass. For the CVar promotion, movement gates use the fixed-on treatment against fixed-off, and the fixed-on arm also clears the L169-relative near/far gates against `ca0bf76`. The A/A and true-baseline runs are measurement controls, not the promotion treatment; the baseline control contained one 16.7 ms worst-tick outlier. Both on/off arms stayed below 14 ms. The per-pair traces, manifests, summaries, and decision are in `Saved/Linux/l172/ue/`. + +The default-on smoke run omitted both command-line CVar overrides. Its backend readback was wait=0 and apply budget=1.0 ms; frame arrival was 1.500 s, p99 6.971 ms, and worst tick 13.052 ms. + +## Defaults, checks, and limits + +| CVar | Final default | +| --- | ---: | +| `voxel.StreamingFastTravelWaitForAllCriticalTiles` | 0 | +| `voxel.StreamingFastTravelApplyBudgetMilliseconds` | 1.0 ms | +| `voxel.CoreRegionConcurrentAdmission` | 1 | +| `voxel.StreamingIncrementalDesiredSet` | 1 | +| `voxel.StreamingFastTravelForwardFirst` | 0 | +| `voxel.StreamingRebuildOnCoreRegionCompletion` | 1 | +| `voxel.StreamingCoreDesiredCacheByCenter` | 0 | +| `voxel.StreamingIncrementalDesiredAuditEvery` | 0 | +| `voxel.StreamingIncrementalDesiredMaterializationAuditEvery` | 0 | +| `voxel.StreamingIncrementalDesiredAuditInjectMismatch` | 0 | + +- Fast tier: **PASS**, 269 checks and 5 non-gating diagnostics, 284.7 s. The new mid-pass refill regression and owner W1/W8/W12 determinism checks passed. +- Final default-on VFHost build: **PASS**, 150.58 s; `VoxelWorld.cpp` compiled and the DWARF sizes were 480/480/480 bytes. +- Default-on owner-world smoke: **PASS**, 52.33 s. Saved L169/L170 rescoring, all three 10-pair batches, runner syntax compilation, and `git diff --check` passed. +- Two failed preflight launches were excluded: UE first inherited the standalone 6 GiB address-space limit, then the runner used an export path outside the plugin Saved directory. An initial fixed-off warm-up produced a valid trace but exposed an overstrict latch assertion; the check was corrected and the final warm-up and measured pairs passed. None of these launches contributed paired timing values. +- UE W12 movement was not run; the harness exposes eight effective tile slots. The timing experiment covers the owner four-stratum plan under Null RHI. + +Changed Unreal `Source/` files: + +- `Source/VoxelForge/Private/VoxelWorld.cpp` +- `Source/VoxelForge/Private/VoxelStreamingAdmissionOrder.h` diff --git a/Standalone/Research/WorldAssemblerM3/Linux172/RunL172Movement.py b/Standalone/Research/WorldAssemblerM3/Linux172/RunL172Movement.py new file mode 100644 index 0000000..4fc0316 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux172/RunL172Movement.py @@ -0,0 +1,868 @@ +#!/usr/bin/env python3 +"""Build isolated L172 UE trees, measure admission, and rescore saved traces.""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import re +import shutil +import statistics +import subprocess +import time +from typing import Any + + +ROOT = Path(__file__).resolve().parents[4] +HOST = Path("/home/jahni/VFHost") +ENGINE = Path("/home/jahni/UnrealEngine") +MAIN_CHECKOUT = Path("/home/jahni/VoxelM/Plugins/VoxelForge") +OUT = ROOT / "Saved/Linux/l172" +UE_OUT = OUT / "ue" +BUILD_OUT = OUT / "builds" +BIN_OUT = OUT / "bin" +HOST_PLUGIN_LINK = HOST / "PluginDeps/VoxelForge" +HOST_PLUGIN_BINARY = HOST / "Binaries/Linux/libUnrealEditor-VoxelForge.so" +EDITOR = ENGINE / "Engine/Binaries/Linux/UnrealEditor-Cmd" +BUILD_SH = ENGINE / "Engine/Build/BatchFiles/Linux/Build.sh" +DWARFDUMP = Path("/usr/lib/llvm-19/bin/llvm-dwarfdump") +INTERMEDIATE = HOST / "Intermediate" +MOVE_START_S = 6.0 +MOVE_DURATION_S = 12.0 +SPAWN_M = (19.166015625, -14.2001953125, -753.3740234375) +LINK_NAMES = ("Source", "Standalone", "VoxelForge.uplugin") +OBJECTS = { + "voxel_world": "External/Build/Linux/x64/UnrealEditor/Development/VoxelForge/VoxelWorld.cpp.o", + "core_adapter": "External/Build/Linux/x64/UnrealEditor/Development/VoxelForge/VoxelForgeCoreAdapter.cpp.o", + "core_api_shim": "External/Build/Linux/x64/UnrealEditor/Development/VoxelForgeCore/VFCore_Core_src_VoxelForgeCore_cpp.cpp.o", +} + +WAIT_OFF = 1 +APPLY_OFF_MS = 0.0 +WAIT_ON = 0 +APPLY_ON_MS = 1.0 + + +def utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + + +def git(root: Path, *args: str) -> str: + result = subprocess.run(["git", "-C", str(root), *args], text=True, + capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"git -C {root} {' '.join(args)} failed: {result.stderr.strip()}") + return result.stdout.rstrip("\n") + + +def root_info(root: Path) -> dict[str, Any]: + actual_root = Path(git(root, "rev-parse", "--show-toplevel")).resolve() + expected_root = root.resolve() + if actual_root != expected_root: + raise RuntimeError(f"plugin root mismatch: requested={expected_root}, git={actual_root}") + return { + "plugin_root": str(actual_root), + "git_head": git(actual_root, "rev-parse", "HEAD"), + "git_status_porcelain": git(actual_root, "status", "--porcelain"), + } + + +def link_roots() -> dict[str, str]: + roots = {} + for name in LINK_NAMES: + link = HOST_PLUGIN_LINK / name + if not link.is_symlink(): + raise RuntimeError(f"VFHost plugin dependency is not a symlink: {link}") + resolved = link.resolve(strict=True) + roots[name] = str(resolved.parent) + return roots + + +def assert_shared_root(expected_root: Path) -> dict[str, str]: + expected = expected_root.resolve(strict=True) + roots = link_roots() + if any(Path(root) != expected for root in roots.values()): + raise RuntimeError(f"VFHost plugin symlinks do not share expected root {expected}: {roots}") + return roots + + +def _move_plugin_intermediates(old_root: str | None, new_root: Path) -> list[dict[str, str]]: + matches = [] + if INTERMEDIATE.exists(): + for current, dirs, _files in os.walk(INTERMEDIATE, topdown=True): + for dirname in list(dirs): + if dirname.startswith("VoxelForge"): + matches.append(Path(current) / dirname) + dirs.remove(dirname) + if not matches: + return [] + head = "unknown" + if old_root: + try: + head = root_info(Path(old_root))["git_head"][:12] + except Exception: + head = Path(old_root).name + archive = OUT / "intermediate-archive" / f"{time.time_ns()}-{head}-to-{new_root.name}" + moved = [] + for source in sorted(matches, key=lambda path: len(path.parts)): + relative = source.relative_to(INTERMEDIATE) + target = archive / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(source), str(target)) + moved.append({"from": str(source), "to": str(target)}) + return moved + + +def switch_plugin_root(target_root: Path) -> dict[str, Any]: + target = target_root.resolve(strict=True) + if not (target / "Source").is_dir() or not (target / "Standalone").is_dir(): + raise RuntimeError(f"tree does not contain its own Source and Standalone: {target}") + if not (target / "VoxelForge.uplugin").is_file(): + raise RuntimeError(f"tree has no plugin descriptor: {target}") + + current_roots = set(link_roots().values()) + old_root = next(iter(current_roots)) if len(current_roots) == 1 else None + moved = [] + if old_root != str(target): + moved = _move_plugin_intermediates(old_root, target) + for name in LINK_NAMES: + link = HOST_PLUGIN_LINK / name + temporary = link.with_name(f"{link.name}.l172-{os.getpid()}.tmp") + if temporary.exists() or temporary.is_symlink(): + temporary.unlink() + temporary.symlink_to(target / name) + os.replace(temporary, link) + roots = assert_shared_root(target) + return {"active_root": str(target), "links": roots, "intermediates_moved": moved} + + +def restore_main_checkout() -> dict[str, Any]: + return switch_plugin_root(MAIN_CHECKOUT) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def dwarf_struct_size(path: Path) -> int: + if not DWARFDUMP.is_file(): + raise RuntimeError(f"llvm-dwarfdump is missing: {DWARFDUMP}") + result = subprocess.run([str(DWARFDUMP), "--name=WorldAssemblyStratumPlanV1", str(path)], + text=True, capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"llvm-dwarfdump failed for {path}: {result.stderr.strip()}") + dies = re.split(r"(?=0x[0-9a-fA-F]+: DW_TAG_)", result.stdout) + matches = [] + for die in dies: + if not re.search(r'DW_AT_name\s+\("WorldAssemblyStratumPlanV1"\)', die): + continue + size = re.search(r"DW_AT_byte_size\s+\(0x([0-9a-fA-F]+)\)", die) + if size: + matches.append(int(size.group(1), 16)) + if len(matches) != 1: + raise RuntimeError(f"expected one complete plan type in {path}; sizes={matches}") + return matches[0] + + +def dwarf_pre_run_gate() -> dict[str, Any]: + sizes = {} + for label, relative in OBJECTS.items(): + path = INTERMEDIATE / relative + if not path.is_file(): + raise RuntimeError(f"pre-run DWARF gate object is missing: {path}") + sizes[label] = {"object": str(path), "byte_size": dwarf_struct_size(path)} + unique = {item["byte_size"] for item in sizes.values()} + if len(unique) != 1: + raise RuntimeError(f"WorldAssemblyStratumPlanV1 size mismatch: {sizes}") + return {"status": "PASS", "struct": "WorldAssemblyStratumPlanV1", "objects": sizes} + + +def create_fresh_baseline_worktree() -> Path: + timestamp = str(time.time_ns()) + repo_host = OUT / "worktree-hosts" / f"ca0bf76-git-{timestamp}" + parent = OUT / "worktrees" + parent.mkdir(parents=True, exist_ok=True) + repo_host.parent.mkdir(parents=True, exist_ok=True) + commit = git(ROOT, "rev-parse", "ca0bf76") + clone = subprocess.run(["git", "clone", "--shared", "--no-checkout", + str(MAIN_CHECKOUT), str(repo_host)], text=True, + capture_output=True, check=False) + if clone.returncode != 0: + raise RuntimeError(f"isolated git worktree host clone failed: {clone.stderr.strip()}") + fetch = subprocess.run(["git", "-C", str(repo_host), "fetch", "--no-tags", + str(MAIN_CHECKOUT), commit], text=True, + capture_output=True, check=False) + if fetch.returncode != 0: + raise RuntimeError(f"ca0bf76 fetch into isolated worktree host failed: " + f"{fetch.stderr.strip()}") + path = parent / f"assembler-world-ca0bf76-fresh-{timestamp}" + result = subprocess.run(["git", "-C", str(repo_host), "worktree", "add", "--detach", + str(path), commit], text=True, capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"fresh ca0bf76 worktree add failed: {result.stderr.strip()}") + info = root_info(path) + if not info["git_head"].startswith("ca0bf767ca7afe40723bf8bb6987d9efc9ea7523"): + raise RuntimeError(f"fresh baseline worktree has unexpected HEAD: {info}") + return path.resolve() + + +def build_tree(root: Path, label: str) -> dict[str, Any]: + switch = switch_plugin_root(root) + source = root_info(root) + if len(set(switch["links"].values())) != 1: + raise RuntimeError(f"link roots diverged before build: {switch}") + log = BUILD_OUT / f"ubt-{label}.log" + log.parent.mkdir(parents=True, exist_ok=True) + command = [str(BUILD_SH), "VFHostEditor", "Linux", "Development", + f"-Project={HOST / 'VFHost.uproject'}", "-NoUBA", "-MaxParallelActions=4", + f"-Log={log}"] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(["taskset", "-c", "0-11", "timeout", "1800s", *command], + cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, + check=False, timeout=1800) + elapsed = time.monotonic() - started + output = log.read_text(encoding="utf-8", errors="replace") + if result.returncode != 0: + raise RuntimeError(f"UE build {label} failed ({result.returncode}); log={log}") + if not re.search(r"Compile VoxelWorld\.cpp", output): + raise RuntimeError(f"UE build {label} did not compile VoxelWorld.cpp; log={log}") + gate = dwarf_pre_run_gate() + if not HOST_PLUGIN_BINARY.is_file(): + raise RuntimeError(f"UE build did not produce {HOST_PLUGIN_BINARY}") + artifact_dir = BIN_OUT / label + artifact_dir.mkdir(parents=True, exist_ok=True) + artifact = artifact_dir / "libUnrealEditor-VoxelForge.so" + shutil.copy2(HOST_PLUGIN_BINARY, artifact) + return { + **source, + "build_command": command, + "build_log": str(log), + "compiled_voxel_world_cpp": True, + "elapsed_s": round(elapsed, 3), + "dwarf_pre_run_gate": gate, + "binary": str(artifact), + "binary_sha256": sha256_file(artifact), + "active_links": switch["links"], + "intermediates_moved_before_build": switch["intermediates_moved"], + } + + +def prepare() -> int: + OUT.mkdir(parents=True, exist_ok=True) + baseline_root = None + builds = {} + try: + baseline_root = create_fresh_baseline_worktree() + builds["ca0bf76"] = build_tree(baseline_root, "ca0bf76") + builds["ca0bf76"]["worktree_root"] = str(baseline_root) + builds["ca0bf76"]["worktree_created_with"] = [ + "git", "worktree", "add", "--detach", str(baseline_root), "ca0bf76"] + write_json(OUT / "builds.json", {"status": "BASELINE_READY", **builds}) + builds["fixed"] = build_tree(ROOT, "fixed") + builds["fixed"]["worktree_root"] = str(ROOT.resolve()) + write_json(OUT / "builds.json", {"status": "BOTH_TREES_READY", **builds}) + print("[L172Builds] " + json.dumps({ + key: {"head": value["git_head"], "root": value["plugin_root"], + "binary_sha256": value["binary_sha256"], + "dwarf_sizes": {name: item["byte_size"] + for name, item in value["dwarf_pre_run_gate"]["objects"].items()}} + for key, value in builds.items() if key != "status"}, sort_keys=True), flush=True) + return 0 + finally: + restored = restore_main_checkout() + write_json(OUT / "restored-main-links.json", { + "time_utc": utc_now(), "main_root": str(MAIN_CHECKOUT.resolve()), + "links": restored["links"], "intermediates_moved": restored["intermediates_moved"]}) + + +def read_events(path: Path) -> list[dict[str, Any]]: + rows = [] + if not path.is_file(): + return rows + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not line.startswith("{"): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def first_frame_arrival(events: list[dict[str, Any]]) -> dict[str, Any] | None: + for event in events: + if event.get("type") != "scheduler_admission": + continue + missing = event.get("desired_missing_by_lod", []) + pending = event.get("pending_by_lod", []) + if (len(missing) >= 3 and len(pending) >= 3 + and sum(int(value) for value in missing[:3]) == 0 + and sum(int(value) for value in pending[:3]) == 0): + return event + return None + + +def percentile(values: list[float], fraction: float = 0.99) -> float | None: + if not values: + return None + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1)) + return ordered[index] + + +def percent(numerator: int | float, denominator: int | float) -> float | None: + return 100.0 * numerator / denominator if denominator > 0 else None + + +def summarize_trial(condition: str, trace: Path, result: subprocess.CompletedProcess[str], + wall_s: float, expected_wait: int, expected_budget_ms: float) -> dict[str, Any]: + events = read_events(trace) + backend = next((event for event in events if event.get("type") == "worker_backend_config"), {}) + scheduler = [event for event in events if event.get("type") == "scheduler_admission"] + if not scheduler: + raise RuntimeError(f"no scheduler_admission readback in {trace}") + if (int(backend.get("streaming_fast_travel_wait_for_all_critical_tiles", -1)) + != expected_wait + or abs(float(backend.get("streaming_fast_travel_apply_budget_ms", -999.0)) + - expected_budget_ms) > 0.01 + or int(backend.get("effective_tile_slots", -1)) != 8): + raise RuntimeError(f"switch or worker readback mismatch in {trace}: {backend}") + if any(int(event.get("forward_first", -1)) != 0 + or int(event.get("rebuild_on_core_region_completion", -1)) != 1 + or int(event.get("core_desired_cache_by_center", -1)) != 0 + or int(event.get("incremental_desired_set", -1)) != 1 + or int(event.get("incremental_desired_audit_every", -1)) != 0 + or int(event.get("incremental_desired_materialization_audit_every", -1)) != 0 + or int(event.get("incremental_desired_audit_inject_mismatch", -1)) != 0 + for event in scheduler): + raise RuntimeError(f"non-default scheduler CVar readback in {trace}") + + arrival = first_frame_arrival(scheduler) + if arrival is None: + raise RuntimeError(f"missing frame-resolution LOD0-2 settled event in {trace}") + if condition == "fixed-off": + arrival_s = float(arrival["t_s"]) + pre_arrival = [event for event in scheduler + if float(event.get("t_s", 0.0)) < arrival_s] + post_arrival = [event for event in scheduler + if float(event.get("t_s", 0.0)) >= arrival_s] + if any(int(event.get("fast_travel_mode", 0)) != 0 for event in pre_arrival): + raise RuntimeError(f"fixed-off entered fast-travel mode before initial arrival: {trace}") + if not any(int(event.get("fast_travel_mode", 0)) == 1 for event in post_arrival): + raise RuntimeError(f"fixed-off never latched fast-travel mode after initial arrival: {trace}") + if condition == "fixed-on": + arrival_s = float(arrival["t_s"]) + pre_arrival = [event for event in scheduler if float(event.get("t_s", 0.0)) < arrival_s] + apply_events = [event for event in events if event.get("type") == "apply_frame"] + if any(int(event.get("fast_travel_mode", 0)) != 0 for event in pre_arrival): + raise RuntimeError(f"fast-travel mode activated before frame-settled arrival: {trace}") + if any(float(event.get("apply_budget_ms", 0.0)) < 1.99 + and float(event.get("t_s", 0.0)) <= arrival_s for event in apply_events): + raise RuntimeError(f"fast-travel apply budget activated before arrival: {trace}") + if not any(int(event.get("initial_arrival_set_settled", 0)) == 1 + and int(event.get("fast_travel_mode", 0)) == 1 + for event in scheduler): + raise RuntimeError(f"fixed-on latch did not activate after settled arrival: {trace}") + if not any(float(event.get("apply_budget_ms", 0.0)) <= 1.001 + for event in apply_events): + raise RuntimeError(f"fixed-on 1 ms apply budget never activated: {trace}") + move_stop = next((event for event in events + if event.get("type") == "streaming_test_move_stopped"), None) + if move_stop is None: + raise RuntimeError(f"missing move-stop event in {trace}") + stop_s = float(move_stop.get("t_s", 0.0)) + samples = [event for event in events if event.get("type") == "streaming_sample"] + moving = [event for event in samples + if MOVE_START_S <= float(event.get("t_s", 0.0)) < stop_s + and float(event.get("speed_cm_s", 0.0)) >= 50.0] + dwell = [event for event in samples + if float(event.get("t_s", 0.0)) >= stop_s + and float(event.get("speed_cm_s", 0.0)) < 50.0] + if result.returncode != 0 or len(moving) < 40 or len(dwell) < 16: + raise RuntimeError(f"incomplete run {trace}: exit={result.returncode}, " + f"moving={len(moving)}, dwell={len(dwell)}") + work_start = min(float(event.get("t_s", 0.0)) for event in moving) + frames = [event for event in events if event.get("type") == "game_thread_work_frame" + and work_start <= float(event.get("t_s", 0.0)) < stop_s] + if not frames: + raise RuntimeError(f"missing moving game-thread frames in {trace}") + near, far_holes = [], [] + for event in moving: + desired = event.get("desired_by_lod", [0] * 9) + resident = event.get("resident_by_lod", [0] * 9) + covered = event.get("covered_by_lod", resident) + near.append(percent(sum(resident[:3]), sum(desired[:3])) or 0.0) + far_holes.append(percent(sum(desired[3:]) - sum(covered[3:]), + sum(desired[3:])) or 0.0) + settled_sample = next((event for event in samples + if float(event.get("t_s", 0.0)) < work_start + and float(event.get("speed_cm_s", 0.0)) < 50.0 + and sum(event.get("settled_by_lod", [0] * 9)[:3]) + >= sum(event.get("desired_by_lod", [0] * 9)[:3]) > 0), None) + return { + "condition": condition, + "moving_samples": len(moving), + "dwell_samples": len(dwell), + "effective_tile_slots": backend.get("effective_tile_slots"), + "frame_resolution_arrival_s": float(arrival["t_s"]), + "frame_resolution_arrival_frame": int(arrival.get("frame", -1)), + "streaming_sample_arrival_s": (float(settled_sample["t_s"]) + if settled_sample else None), + "near_readiness_moving_mean_pct": statistics.mean(near), + "far_holes_moving_median_pct": statistics.median(far_holes), + "tick_p99_ms": percentile([float(frame.get("tick_ms", 0.0)) for frame in frames]), + "tick_worst_ms": max(float(frame.get("tick_ms", 0.0)) for frame in frames), + "early_admission_wait_for_critical": expected_wait, + "early_admission_apply_budget_ms": expected_budget_ms, + "trace": str(trace.relative_to(ROOT)), + "wall_s": round(wall_s, 3), + } + + +def run_trial(name: str, batch: str, condition: str, root: Path, binary: Path, + wait: int, apply_ms: float, + include_switch_overrides: bool = True) -> dict[str, Any]: + switch = switch_plugin_root(root) + identity = root_info(root) + assert_shared_root(root) + if not binary.is_file(): + raise FileNotFoundError(binary) + binary_hash = sha256_file(binary) + shutil.copy2(binary, HOST_PLUGIN_BINARY) + run_dir = UE_OUT / batch / condition + run_dir.mkdir(parents=True, exist_ok=True) + export = HOST / "Plugins/VoxelForge/Saved/Linux/l172/export" / batch / condition / name + userdir = HOST / "Saved/Linux/l172/user" / batch / condition / name + zendata = HOST / "Saved/Linux/l172/zen" / batch / condition / name + ddc = HOST / "Saved/Linux/l172/DDC" + for directory in (export, userdir, zendata, ddc): + directory.mkdir(parents=True, exist_ok=True) + trace = run_dir / f"{name}.trace.jsonl" + log = run_dir / f"{name}.log" + stdout = run_dir / f"{name}.stdout.log" + manifest_path = run_dir / f"{name}.manifest.json" + trace.unlink(missing_ok=True) + spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M) + command = [ + str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", + "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", + "-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer", + "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", + f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", + "-archetype=TunnelNetwork", "-slot=0", "-modes=export", "-gameconfig=1", + "-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25", + "-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", + "-coreplantest=0", "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", + "-corehybridmesher=0", f"-out={export}", + "-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=3000.0", + f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}", + f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}", + "-voxel.TestExitSeconds=0.0", "-voxel.TestMoveDirection=1,0,0", + "-voxel.TestExitOnSteady=0", "-coreworldstreamtrace=1", + f"-voxel.StartupTraceSpawnCm={spawn_cm}", + "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", + "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", + f"-abslog={log}", + ] + if include_switch_overrides: + command.extend(( + f"-voxel.StreamingFastTravelWaitForAllCriticalTiles={wait}", + f"-voxel.StreamingFastTravelApplyBudgetMilliseconds={apply_ms:.1f}", + )) + manifest = { + **identity, + "created_utc": utc_now(), + "batch": batch, + "condition": condition, + "run": name, + "active_links": switch["links"], + "intermediates_moved_on_tree_switch": switch["intermediates_moved"], + "binary": str(binary), + "binary_sha256": binary_hash, + "staged_host_binary_sha256": sha256_file(HOST_PLUGIN_BINARY), + "expected_switches": {"wait_for_all_critical_tiles": wait, + "apply_budget_ms": apply_ms}, + "switch_source": ("command_line_overrides" if include_switch_overrides + else "compiled_defaults"), + "command": command, + "trace": str(trace), + "unreal_log": str(log), + } + write_json(manifest_path, manifest) + started = time.monotonic() + with stdout.open("w", encoding="utf-8") as stream: + result = subprocess.run(["taskset", "-c", "0-11", "timeout", "1740s", *command], + cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, + check=False, timeout=1740) + wall_s = time.monotonic() - started + try: + row = summarize_trial(condition, trace, result, wall_s, wait, apply_ms) + row["run"] = name + manifest["result"] = {"exit_code": result.returncode, "wall_s": round(wall_s, 3), + "status": "PASS", "owner_world_ran": True} + row["manifest"] = str(manifest_path.relative_to(ROOT)) + write_json(manifest_path, manifest) + return row + except Exception as error: + manifest["result"] = {"exit_code": result.returncode, "wall_s": round(wall_s, 3), + "status": "FAIL", "error": str(error)} + write_json(manifest_path, manifest) + raise + + +def frame_arrival_from_trace(path: Path) -> float | None: + event = first_frame_arrival(read_events(path)) + return float(event["t_s"]) if event is not None else None + + +def rescore_saved_traces() -> dict[str, Any]: + result: dict[str, Any] = {"metric": "first scheduler_admission with " + "sum(desired_missing_by_lod[0:3]) == 0 and " + "sum(pending_by_lod[0:3]) == 0; t_s at that scheduler frame"} + for round_no in ("169", "170"): + root = ROOT / f"Saved/Linux/l{round_no}/ue/movement" + arms = {} + for arm in ("baseline", "candidate"): + values = {} + for trace in sorted((root / arm).glob("pair-*.trace.jsonl")): + arrival = frame_arrival_from_trace(trace) + if arrival is not None: + number = int(re.search(r"pair-(\d+)", trace.name).group(1)) + values[number] = {"arrival_s": arrival, "trace": str(trace.relative_to(ROOT))} + arms[arm] = values + pairs = [] + for number in sorted(arms["baseline"].keys() & arms["candidate"].keys()): + delta = arms["candidate"][number]["arrival_s"] - arms["baseline"][number]["arrival_s"] + pairs.append({"pair": number, "delta_s_candidate_minus_baseline": delta, + "candidate_slower": delta > 0, + "baseline": arms["baseline"][number], + "candidate": arms["candidate"][number]}) + deltas = [pair["delta_s_candidate_minus_baseline"] for pair in pairs] + result[round_no] = { + "pairs": pairs, + "baseline_median_arrival_s": statistics.median( + [value["arrival_s"] for value in arms["baseline"].values()]) + if arms["baseline"] else None, + "candidate_median_arrival_s": statistics.median( + [value["arrival_s"] for value in arms["candidate"].values()]) + if arms["candidate"] else None, + "paired_median_delta_ms": 1000.0 * statistics.median(deltas) if deltas else None, + "candidate_slower_pairs": sum(delta > 0 for delta in deltas), + "pair_count": len(pairs), + } + path = OUT / "arrival-rescore-l169-l170.json" + write_json(path, result) + print("[L172ArrivalRescore] " + json.dumps({ + key: {metric: value for metric, value in data.items() if metric != "pairs"} + for key, data in result.items() if key in ("169", "170")}, sort_keys=True), flush=True) + return result + + +def percentile_nearest_rank(values: list[float], fraction: float) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("percentile requires values") + return ordered[max(0, min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1))] + + +def paired_deltas(rows: list[dict[str, Any]], first: str, second: str, + metric: str) -> list[dict[str, Any]]: + by_pair: dict[int, dict[str, dict[str, Any]]] = {} + for row in rows: + pair_match = re.search(r"pair-(\d+)", row["run"]) + if not pair_match: + continue + pair = int(pair_match.group(1)) + by_pair.setdefault(pair, {})[row["condition"]] = row + result = [] + for pair, arms in sorted(by_pair.items()): + if first not in arms or second not in arms: + continue + a, b = arms[first][metric], arms[second][metric] + result.append({"pair": pair, "first": first, "second": second, + "delta_s_second_minus_first": b - a, + "first_value": a, "second_value": b}) + return result + + +def run_batch_arm(rows: list[dict[str, Any]], batch: str, condition: str, + root: Path, binary: Path, wait: int, apply_ms: float, + run_name: str) -> dict[str, Any]: + row = run_trial(run_name, batch, condition, root, binary, wait, apply_ms) + rows.append(row) + write_json(UE_OUT / batch / "runs-partial.json", rows) + return row + + +def load_partial_rows(batch: str) -> list[dict[str, Any]]: + path = UE_OUT / batch / "runs-partial.json" + if not path.is_file(): + return [] + rows = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(rows, list): + raise RuntimeError(f"invalid partial rows file: {path}") + return rows + + +def has_run(rows: list[dict[str, Any]], condition: str, run_name: str) -> bool: + return any(row.get("condition") == condition and row.get("run") == run_name + for row in rows) + + +def require_quiet_machine() -> dict[str, Any]: + uptime = subprocess.run(["uptime"], text=True, capture_output=True, check=False).stdout.strip() + load = os.getloadavg() + process_list = subprocess.run(["ps", "-eo", "args="], text=True, capture_output=True, + check=False).stdout.splitlines() + busy = [line.strip() for line in process_list + if any(token in line for token in ("UnrealBuildTool", "Build.sh", "UnrealEditor-Cmd"))] + record = {"uptime": uptime, "load_1_5_15": load, "other_build_or_editor_processes": busy, + "checked_utc": utc_now()} + if load[0] >= 1.5 or load[1] >= 1.5 or busy: + raise RuntimeError(f"machine is not quiet enough for timing: {record}") + return record + + +def run_matrix() -> int: + builds_path = OUT / "builds.json" + if not builds_path.is_file(): + raise RuntimeError("missing build manifests; run this script prepare first") + builds = json.loads(builds_path.read_text(encoding="utf-8")) + fixed_root = Path(builds["fixed"]["plugin_root"]) + baseline_root = Path(builds["ca0bf76"]["plugin_root"]) + fixed_binary = Path(builds["fixed"]["binary"]) + baseline_binary = Path(builds["ca0bf76"]["binary"]) + for build in (builds["fixed"], builds["ca0bf76"]): + if not build.get("compiled_voxel_world_cpp"): + raise RuntimeError("build manifest lacks a full VoxelWorld.cpp compile") + sizes = {item["byte_size"] for item in build["dwarf_pre_run_gate"]["objects"].values()} + if len(sizes) != 1: + raise RuntimeError(f"saved DWARF gate failed: {build['plugin_root']} {sizes}") + if sha256_file(fixed_binary) != builds["fixed"]["binary_sha256"]: + raise RuntimeError("fixed binary does not match its build manifest") + if sha256_file(baseline_binary) != builds["ca0bf76"]["binary_sha256"]: + raise RuntimeError("ca0bf76 binary does not match its build manifest") + + quiet = require_quiet_machine() + write_json(OUT / "timing-machine.json", quiet) + all_rows: dict[str, list[dict[str, Any]]] = { + "aa": load_partial_rows("aa"), + "off_vs_ca0bf76": load_partial_rows("off-vs-ca0bf76"), + "on_vs_off": load_partial_rows("on-vs-off"), + } + try: + # A/A arms use the exact same staged fixed binary and the same CVar values. + aa = all_rows["aa"] + for arm in ("aa-a", "aa-b"): + if not has_run(aa, arm, "warmup"): + run_batch_arm(aa, "aa", arm, fixed_root, fixed_binary, WAIT_OFF, APPLY_OFF_MS, + "warmup") + for pair in range(1, 11): + for condition in ("aa-a", "aa-b"): + if not has_run(aa, condition, f"pair-{pair}"): + run_batch_arm(aa, "aa", condition, fixed_root, fixed_binary, + WAIT_OFF, APPLY_OFF_MS, f"pair-{pair}") + aa_pairs = paired_deltas(aa, "aa-a", "aa-b", "frame_resolution_arrival_s") + aa_abs_p90_ms = 1000.0 * percentile_nearest_rank( + [abs(item["delta_s_second_minus_first"]) for item in aa_pairs], 0.90) + aa_summary = {"pairs": aa_pairs, "pair_count": len(aa_pairs), + "absolute_delta_p90_ms": aa_abs_p90_ms} + write_json(UE_OUT / "aa" / "summary.json", aa_summary) + + off_vs_base = all_rows["off_vs_ca0bf76"] + for condition, root, binary in (("fixed-off", fixed_root, fixed_binary), + ("ca0bf76", baseline_root, baseline_binary)): + if not has_run(off_vs_base, condition, "warmup"): + run_batch_arm(off_vs_base, "off-vs-ca0bf76", condition, root, binary, + WAIT_OFF, APPLY_OFF_MS, "warmup") + for pair in range(1, 11): + control_order = (("fixed-off", "ca0bf76") if pair % 2 + else ("ca0bf76", "fixed-off")) + for condition in control_order: + if has_run(off_vs_base, condition, f"pair-{pair}"): + continue + root, binary = ((fixed_root, fixed_binary) if condition == "fixed-off" + else (baseline_root, baseline_binary)) + run_batch_arm(off_vs_base, "off-vs-ca0bf76", condition, root, binary, + WAIT_OFF, APPLY_OFF_MS, f"pair-{pair}") + control_pairs = paired_deltas(off_vs_base, "ca0bf76", "fixed-off", + "frame_resolution_arrival_s") + control_slower = sum(item["delta_s_second_minus_first"] > 0 for item in control_pairs) + control_summary = {"pairs": control_pairs, "pair_count": len(control_pairs), + "fixed_off_slower_pairs": control_slower, + "stop_at_eight_slower": control_slower >= 8} + write_json(UE_OUT / "off-vs-ca0bf76" / "summary.json", control_summary) + if control_slower >= 8: + final = {"status": "STOPPED_CONTROL_FAILED", "timing_machine": quiet, + "aa": aa_summary, "fixed_off_vs_ca0bf76": control_summary, + "fixed_on_vs_off": None, + "reason": "fixed-off was slower than true ca0bf76 in at least 8/10 pairs"} + write_json(UE_OUT / "decision-summary.json", final) + print("[L172Decision] " + json.dumps({key: value for key, value in final.items() + if key not in ("aa", "fixed_off_vs_ca0bf76")}, + sort_keys=True), flush=True) + return 0 + + on_vs_off = all_rows["on_vs_off"] + for condition, wait, budget in (("fixed-on", WAIT_ON, APPLY_ON_MS), + ("fixed-off", WAIT_OFF, APPLY_OFF_MS)): + if not has_run(on_vs_off, condition, "warmup"): + run_batch_arm(on_vs_off, "on-vs-off", condition, fixed_root, fixed_binary, + wait, budget, "warmup") + settings = {"fixed-on": (WAIT_ON, APPLY_ON_MS), + "fixed-off": (WAIT_OFF, APPLY_OFF_MS)} + for pair in range(1, 11): + on_off_order = (("fixed-on", "fixed-off") if pair % 2 + else ("fixed-off", "fixed-on")) + for condition in on_off_order: + if has_run(on_vs_off, condition, f"pair-{pair}"): + continue + wait, budget = settings[condition] + run_batch_arm(on_vs_off, "on-vs-off", condition, fixed_root, fixed_binary, + wait, budget, f"pair-{pair}") + promotion_pairs = paired_deltas(on_vs_off, "fixed-off", "fixed-on", + "frame_resolution_arrival_s") + arrival_deltas_ms = [1000.0 * item["delta_s_second_minus_first"] + for item in promotion_pairs] + paired_median_delta_ms = statistics.median(arrival_deltas_ms) + slower_pairs = sum(value > 0 for value in arrival_deltas_ms) + tolerance_ms = max(aa_abs_p90_ms, 17.0) + fixed_off_rows = [row for row in on_vs_off if row["condition"] == "fixed-off" + and row["run"].startswith("pair-")] + fixed_on_rows = [row for row in on_vs_off if row["condition"] == "fixed-on" + and row["run"].startswith("pair-")] + near_gain = (statistics.median(row["near_readiness_moving_mean_pct"] + for row in fixed_on_rows) + - statistics.median(row["near_readiness_moving_mean_pct"] + for row in fixed_off_rows)) + far_change = (statistics.median(row["far_holes_moving_median_pct"] + for row in fixed_on_rows) + - statistics.median(row["far_holes_moving_median_pct"] + for row in fixed_off_rows)) + moving_gate_rows = [row for row in on_vs_off if row["run"].startswith("pair-")] + gates = { + "arrival_median_delta_within_tolerance": paired_median_delta_ms <= tolerance_ms, + "arrival_candidate_slower_in_at_most_6_of_10": slower_pairs <= 6, + "near_gain_at_least_9_6_pp": near_gain >= 9.6, + "far_hole_change_at_most_minus_1_35_pp": far_change <= -1.35, + "on_off_moving_runs_p99_at_most_11_ms": all( + row["tick_p99_ms"] is not None and row["tick_p99_ms"] <= 11.0 + for row in moving_gate_rows), + "on_off_moving_runs_worst_at_most_14_ms": all( + row["tick_worst_ms"] is not None and row["tick_worst_ms"] <= 14.0 + for row in moving_gate_rows), + } + on_off_summary = { + "pairs": promotion_pairs, + "paired_median_delta_ms": paired_median_delta_ms, + "candidate_slower_pairs": slower_pairs, + "arrival_tolerance_ms": tolerance_ms, + "aa_absolute_delta_p90_ms": aa_abs_p90_ms, + "near_gain_pp": near_gain, + "far_hole_change_pp": far_change, + "gates": gates, + } + write_json(UE_OUT / "on-vs-off" / "summary.json", on_off_summary) + promote = all(gates.values()) and not control_summary["stop_at_eight_slower"] + final = { + "status": "PROMOTE" if promote else "KEEP_DEFAULTS_OFF", + "timing_machine": quiet, + "aa": aa_summary, + "fixed_off_vs_ca0bf76": control_summary, + "fixed_on_vs_off": on_off_summary, + "moving_metrics_medians": { + "fixed_off_near_pct": statistics.median( + row["near_readiness_moving_mean_pct"] for row in fixed_off_rows), + "fixed_on_near_pct": statistics.median( + row["near_readiness_moving_mean_pct"] for row in fixed_on_rows), + "fixed_off_far_holes_pct": statistics.median( + row["far_holes_moving_median_pct"] for row in fixed_off_rows), + "fixed_on_far_holes_pct": statistics.median( + row["far_holes_moving_median_pct"] for row in fixed_on_rows), + "fixed_on_p99_ms_max": max(row["tick_p99_ms"] for row in fixed_on_rows), + "fixed_on_worst_tick_ms_max": max(row["tick_worst_ms"] for row in fixed_on_rows), + }, + "promotion_gates": gates, + "all_runs": all_rows, + "switch_defaults": { + "voxel.StreamingFastTravelWaitForAllCriticalTiles": 0 if promote else 1, + "voxel.StreamingFastTravelApplyBudgetMilliseconds": 1.0 if promote else 0.0, + "voxel.StreamingIncrementalDesiredSet": 1, + "voxel.StreamingFastTravelForwardFirst": 0, + "voxel.StreamingRebuildOnCoreRegionCompletion": 1, + "voxel.StreamingCoreDesiredCacheByCenter": 0, + "voxel.StreamingIncrementalDesiredAuditEvery": 0, + "voxel.StreamingIncrementalDesiredMaterializationAuditEvery": 0, + "voxel.StreamingIncrementalDesiredAuditInjectMismatch": 0, + }, + } + write_json(UE_OUT / "decision-summary.json", final) + print("[L172Decision] " + json.dumps({ + "status": final["status"], "aa_abs_p90_ms": aa_abs_p90_ms, + "arrival_median_delta_ms": paired_median_delta_ms, + "arrival_tolerance_ms": tolerance_ms, "slower_pairs": slower_pairs, + "near_gain_pp": near_gain, "far_hole_change_pp": far_change, + "gates": gates}, sort_keys=True), flush=True) + return 0 + finally: + restored = restore_main_checkout() + write_json(OUT / "restored-main-links.json", { + "time_utc": utc_now(), "main_root": str(MAIN_CHECKOUT.resolve()), + "links": restored["links"], "intermediates_moved": restored["intermediates_moved"]}) + + +def run_default_smoke() -> int: + build = json.loads((OUT / "build-fixed-default-on.json").read_text(encoding="utf-8")) + try: + row = run_trial("default-smoke", "default-on-smoke", "fixed-on", ROOT, + Path(build["binary"]), WAIT_ON, APPLY_ON_MS, + include_switch_overrides=False) + write_json(UE_OUT / "default-on-smoke" / "summary.json", row) + print("[L172DefaultSmoke] " + json.dumps(row, sort_keys=True), flush=True) + return 0 + finally: + restored = restore_main_checkout() + write_json(OUT / "restored-main-links.json", { + "time_utc": utc_now(), "main_root": str(MAIN_CHECKOUT.resolve()), + "links": restored["links"], + "intermediates_moved": restored["intermediates_moved"]}) + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("prepare", help="make a fresh ca0bf76 worktree and clean UE builds") + subparsers.add_parser("rescore", help="rescore saved L169 and L170 traces by scheduler frame") + subparsers.add_parser("matrix", help="run the L172 A/A and counterbalanced 10-pair matrices") + subparsers.add_parser("default-smoke", help="run the built default-on binary without CVar overrides") + args = parser.parse_args() + if args.command == "prepare": + return prepare() + if args.command == "rescore": + rescore_saved_traces() + return 0 + if args.command == "matrix": + return run_matrix() + if args.command == "default-smoke": + return run_default_smoke() + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux172/RunStreamingAdmissionOrderTest.py b/Standalone/Research/WorldAssemblerM3/Linux172/RunStreamingAdmissionOrderTest.py new file mode 100644 index 0000000..61d9df4 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux172/RunStreamingAdmissionOrderTest.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import shutil +import subprocess + + +ROOT = Path(__file__).resolve().parents[4] +SOURCE = ROOT / "Source/VoxelForge/Private/VoxelWorld.cpp" +HEADER_DIR = ROOT / "Source/VoxelForge/Private" +TEST_SOURCE = Path(__file__).with_name("TestStreamingAdmissionOrder.cpp") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + output = args.out.resolve() + output.mkdir(parents=True, exist_ok=True) + + source = SOURCE.read_text(encoding="utf-8") + required = ( + "VoxelStreamingAdmissionOrder::StableSortCandidates(", + "LevelTiles.GetData(), LevelTiles.GetData() + LevelTiles.Num()", + "VoxelStreamingAdmissionOrder::AdmitInOrder(LevelTiles", + ) + if any(token not in source for token in required): + raise SystemExit("VoxelWorld.cpp is not using the tested sort/admission helpers") + if ("LevelSubmissionCount" in source or "LevelTiles.SetNum" in source + or "std::partial_sort(" in source): + raise SystemExit("pass-start truncation returned to the production scheduler") + + compiler = os.environ.get("CXX") or shutil.which("clang++-19") or shutil.which("clang++") + if not compiler: + raise SystemExit("clang++ is required for the streaming admission regression") + executable = output / "StreamingAdmissionOrderTest" + build_log = output / "streaming-admission-order-build.log" + run_log = output / "streaming-admission-order.log" + command = [compiler, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", + f"-I{HEADER_DIR}", str(TEST_SOURCE), "-o", str(executable)] + built = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False) + build_log.write_text(built.stdout + built.stderr, encoding="utf-8") + if built.returncode != 0: + raise SystemExit(f"admission regression compile failed; see {build_log}") + ran = subprocess.run([str(executable)], cwd=ROOT, text=True, capture_output=True, + check=False) + run_log.write_text(ran.stdout + ran.stderr, encoding="utf-8") + if ran.returncode != 0: + raise SystemExit(f"admission regression failed; see {run_log}") + print(ran.stdout.strip()) + print(f"[StreamingAdmissionOrder] production_source_guard=PASS build_log={build_log}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux172/TestStreamingAdmissionOrder.cpp b/Standalone/Research/WorldAssemblerM3/Linux172/TestStreamingAdmissionOrder.cpp new file mode 100644 index 0000000..91e891f --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux172/TestStreamingAdmissionOrder.cpp @@ -0,0 +1,97 @@ +#include "VoxelStreamingAdmissionOrder.h" + +#include +#include +#include +#include + +struct FCandidate +{ + int Priority; + int Id; + int Level; +}; + +static bool LessCandidate(const FCandidate& A, const FCandidate& B) +{ + if (A.Priority != B.Priority) return A.Priority < B.Priority; + return A.Id < B.Id; +} + +static std::vector RunPass(std::vector& Candidates, + int InitiallyActive, int Capacity) +{ + int Active = InitiallyActive; + bool FreedOneSlot = false; + std::vector Submitted; + VoxelStreamingAdmissionOrder::AdmitInOrder( + Candidates, + [&]() { return Active >= Capacity; }, + [&](const FCandidate& Candidate) + { + if (Candidate.Level != 0) + { + std::cerr << "candidate escaped its level pass\n"; + std::exit(2); + } + Submitted.push_back(Candidate.Id); + ++Active; + if (!FreedOneSlot && Submitted.size() == 1) + { + --Active; // A worker finishes one task while this level pass is still running. + FreedOneSlot = true; + } + }, + []() {}); + return Submitted; +} + +int main() +{ + const std::vector Input = { + {4, 4, 0}, {0, 0, 0}, {3, 3, 0}, {1, 1, 0}, {2, 2, 0}}; + constexpr int InitiallyActive = 1; + constexpr int Capacity = 2; + const int PassStartFreeSlots = Capacity - InitiallyActive; + + std::vector FullSort = Input; + std::stable_sort(FullSort.begin(), FullSort.end(), LessCandidate); + const std::vector Expected = {FullSort[0].Id, FullSort[1].Id}; + + std::vector FixedCandidates = Input; + VoxelStreamingAdmissionOrder::StableSortCandidates( + FixedCandidates.begin(), FixedCandidates.end(), LessCandidate); + const std::vector FixedSubmitted = + RunPass(FixedCandidates, InitiallyActive, Capacity); + if (FixedSubmitted != Expected || FixedSubmitted.size() != 2 + || FixedCandidates[1].Level != FixedCandidates[0].Level) + { + std::cerr << "fixed submission order/refill mismatch\n"; + return 1; + } + + // Negative control: the old pass-start partial_sort + SetNum truncation loses the + // second candidate even though a slot frees after the first submission. + std::vector OldCandidates = Input; + const std::size_t InitialCount = std::min( + OldCandidates.size(), static_cast(PassStartFreeSlots)); + std::partial_sort(OldCandidates.begin(), OldCandidates.begin() + InitialCount, + OldCandidates.end(), LessCandidate); + OldCandidates.resize(InitialCount); + const std::vector OldSubmitted = + RunPass(OldCandidates, InitiallyActive, Capacity); + if (OldSubmitted == Expected) + { + std::cerr << "negative control did not reproduce the lost refill\n"; + return 1; + } + + std::cout << "[StreamingAdmissionOrder] full_order="; + for (const int Id : Expected) std::cout << Id; + std::cout << " fixed_submissions="; + for (const int Id : FixedSubmitted) std::cout << Id; + std::cout << " old_submissions="; + for (const int Id : OldSubmitted) std::cout << Id; + std::cout << " same_level_refill=PASS negative_control=PASS status=PASS\n"; + return 0; +} diff --git a/Standalone/Tools/verify/verify.py b/Standalone/Tools/verify/verify.py index 167da39..a885e01 100644 --- a/Standalone/Tools/verify/verify.py +++ b/Standalone/Tools/verify/verify.py @@ -3230,6 +3230,20 @@ def _tier_deadline(_signum: int, _frame: object) -> None: f"{ARGS.tier} tier exceeded its {ARGS.overall_timeout}s overall timeout") +def run_streaming_admission_order_regression(runner: VerificationRunner) -> None: + script = ROOT / "Standalone/Research/WorldAssemblerM3/Linux172/RunStreamingAdmissionOrderTest.py" + log = OUT / "streaming-admission-order.log" + command = [sys.executable, str(script), "--out", str(OUT / "streaming-admission-order")] + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, + check=False, timeout=120) + output = log.read_text(encoding="utf-8", errors="replace") + runner.require("streaming-admission-order-mid-pass-refill", + result.returncode == 0 and "same_level_refill=PASS" in output + and "negative_control=PASS" in output, + f"log={log}") + + def main() -> int: runner = VerificationRunner(ARGS.tier) started = time.monotonic() @@ -3251,6 +3265,7 @@ def main() -> int: if ARGS.tier == "sanitize": run_sanitizer_stage(runner) elif ARGS.tier == "fast": + run_streaming_admission_order_regression(runner) owners = run_owner_passages(runner, (CLANG,)) new_plan_failure = None try: