From b06397df9c1e0099295ca25f754d8080e62d29ce Mon Sep 17 00:00:00 2001 From: Fr0zka Date: Tue, 27 Oct 2026 21:39:21 +0100 Subject: [PATCH] streaming: speed-aware scheduler with hysteresis (L80); arrival near-first, fast travel keeps the horizon Policy 3 (headless + Unreal): walkable ring always first; at rest/arrival LOD0-2 before far; walking interleaves near:far ~4:1 toward heading; fast travel admits far coarse tiles right after the walkable ring. Hysteresis: walking 0.75/0.35 m/s, fast 12/8 m/s. Headless, far holes while moving 30 m/s: L77 9.0% / L79 77.2% / L80 8.9%. Cold LOD0-2 ready (one clean protocol, medians): SW 1.30, FI 3.61, TN 3.07, VS 1.43 s. Unreal 30 m/s traces end 2,233/2,237 loaded (commandlet readiness exit 1) for all policies. Passage zero-padding cut: ~50% fewer passage iterations on active tiles, max 6.6 cm surface displacement, no sign flips; not adopted yet (padding sweep next). World unchanged (v213). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1 --- Source/VoxelForge/Private/VoxelWorld.cpp | 157 +++++++++++++++-- Source/VoxelForge/Public/VoxelWorld.h | 3 + .../WorldAssemblerM3/Linux80/REPORT.md | 73 ++++++++ .../Linux80/RunL80HeadlessMovingCamera.py | 115 ++++++++++++ .../Linux80/RunL80PassageCensus.py | 92 ++++++++++ .../Linux80/RunL80TimingPairs.py | 98 +++++++++++ .../Linux80/RunL80UnrealMovingCamera.py | 164 ++++++++++++++++++ Standalone/Source/StreamBenchmarkHeadless.cpp | 8 +- Standalone/Source/StreamBenchmarkPipeline.h | 22 ++- Standalone/Source/StreamingWorld.h | 123 ++++++++++++- 10 files changed, 830 insertions(+), 25 deletions(-) create mode 100644 Standalone/Research/WorldAssemblerM3/Linux80/REPORT.md create mode 100644 Standalone/Research/WorldAssemblerM3/Linux80/RunL80HeadlessMovingCamera.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux80/RunL80PassageCensus.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux80/RunL80TimingPairs.py create mode 100644 Standalone/Research/WorldAssemblerM3/Linux80/RunL80UnrealMovingCamera.py diff --git a/Source/VoxelForge/Private/VoxelWorld.cpp b/Source/VoxelForge/Private/VoxelWorld.cpp index fe50717..4fd18e7 100644 --- a/Source/VoxelForge/Private/VoxelWorld.cpp +++ b/Source/VoxelForge/Private/VoxelWorld.cpp @@ -208,11 +208,11 @@ namespace GVoxelForgeStreamingCoarseFirst, TEXT("Prioritize walkable tiles, then finish LOD0-2 before far clipmap work; controls far-level order.")); - int32 GVoxelForgeStreamingSchedulerPolicy = 2; + int32 GVoxelForgeStreamingSchedulerPolicy = 3; FAutoConsoleVariableRef CVarVoxelForgeStreamingSchedulerPolicy( TEXT("voxel.StreamingSchedulerPolicy"), GVoxelForgeStreamingSchedulerPolicy, - TEXT("Streaming order: 0=L77 coarse-first, 1=L78 strict near-first, 2=L79 fair far share.")); + TEXT("Streaming order: 0=L77 coarse-first, 1=L78 strict near-first, 2=L79 fair share, 3=L80 speed-aware.")); int32 GVoxelForgeStreamingDynamicNearReserve = 1; FAutoConsoleVariableRef CVarVoxelForgeStreamingDynamicNearReserve( @@ -2931,6 +2931,7 @@ void AVoxelWorld::ConfigureHeadlessStreamingTest() HeadlessStreamingTestDistanceCm = 0.0; HeadlessStreamingTestSpeedCmPerSecond = 0.0f; HeadlessStreamingTestStartSeconds = 0.0f; + HeadlessStreamingTestMoveDurationSeconds = 0.0f; HeadlessStreamingTestExitSeconds = 0.0f; HeadlessStreamingTestTeleportAtSeconds = 0.0f; HeadlessStreamingTestTeleportDistanceCm = 0.0f; @@ -3140,10 +3141,12 @@ void AVoxelWorld::ConfigureHeadlessStreamingTest() || FParse::Param(CommandLine, TEXT("voxel.TestMove")); float Speed = 800.0f; float StartSeconds = 20.0f; + float MoveDurationSeconds = 0.0f; float ExitSeconds = 0.0f; int32 ExitOnSteadyValue = 0; FParse::Value(CommandLine, TEXT("voxel.TestMoveSpeedCmPerSecond="), Speed); FParse::Value(CommandLine, TEXT("voxel.TestMoveStartSeconds="), StartSeconds); + FParse::Value(CommandLine, TEXT("voxel.TestMoveDurationSeconds="), MoveDurationSeconds); FParse::Value(CommandLine, TEXT("voxel.TestExitSeconds="), ExitSeconds); FParse::Value(CommandLine, TEXT("voxel.TestTeleportAfterSeconds="), HeadlessStreamingTestTeleportAtSeconds); @@ -3313,6 +3316,7 @@ void AVoxelWorld::ConfigureHeadlessStreamingTest() } HeadlessStreamingTestSpeedCmPerSecond = FMath::Max(Speed, 0.0f); HeadlessStreamingTestStartSeconds = FMath::Max(StartSeconds, 0.0f); + HeadlessStreamingTestMoveDurationSeconds = FMath::Max(MoveDurationSeconds, 0.0f); bHeadlessStreamingTestMovement = HeadlessStreamingTestSpeedCmPerSecond > 0.0f; } @@ -3323,10 +3327,11 @@ void AVoxelWorld::ConfigureHeadlessStreamingTest() HeadlessStreamingTestBeginSeconds = FPlatformTime::Seconds(); UE_LOG(LogTemp, Display, TEXT("[VoxelForgeStreamingTest] clean=1 movement=%d speed_cm_s=%.3f start_s=%.3f " - "exit_s=%.3f direction=(%.3f,%.3f,%.3f)"), + "move_duration_s=%.3f exit_s=%.3f direction=(%.3f,%.3f,%.3f)"), bHeadlessStreamingTestMovement ? 1 : 0, HeadlessStreamingTestSpeedCmPerSecond, HeadlessStreamingTestStartSeconds, + HeadlessStreamingTestMoveDurationSeconds, HeadlessStreamingTestExitSeconds, HeadlessStreamingTestDirection.X, HeadlessStreamingTestDirection.Y, @@ -4394,6 +4399,26 @@ void AVoxelWorld::AdvanceHeadlessStreamingTest( return; } + if (HeadlessStreamingTestMoveDurationSeconds > 0.0f + && ElapsedSeconds - static_cast(HeadlessStreamingTestStartSeconds) + >= static_cast(HeadlessStreamingTestMoveDurationSeconds)) + { + bHeadlessStreamingTestMovement = false; + InOutPlayerHeading = FVector::ZeroVector; + HeadlessStreamingTestLastElapsedSeconds = ElapsedSeconds; + if (VoxelForgeStartupTrace::IsActive()) + { + VoxelForgeStartupTrace::RecordEvent(TEXT("streaming_test_move_stopped"), + FString::Printf(TEXT("\"move_duration_s\":%s,\"elapsed_s\":%s"), + *FString::SanitizeFloat(HeadlessStreamingTestMoveDurationSeconds), + *FString::SanitizeFloat(ElapsedSeconds))); + } + UE_LOG(LogTemp, Display, + TEXT("[VoxelForgeStreamingTest] movement stopped after %.3f s"), + HeadlessStreamingTestMoveDurationSeconds); + return; + } + if (!bHeadlessStreamingTestTeleportDone && HeadlessStreamingTestTeleportAtSeconds > 0.0f && ElapsedSeconds - static_cast(HeadlessStreamingTestStartSeconds) @@ -11941,7 +11966,35 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw RecordSchedulerSkip(TEXT("core_region_compile_pending")); return; } - const bool bPawnMoving = PlayerHeading.Size() >= 50.0f; + const int32 SchedulerPolicy = FMath::Clamp(GVoxelForgeStreamingSchedulerPolicy, 0, 3); + const float SchedulerSpeedCmPerSecond = PlayerHeading.Size(); + if (SchedulerPolicy == 3) + { + // Schmitt thresholds keep a noisy velocity estimate from switching modes every frame. + if (!bSpeedAwareTravelMode && SchedulerSpeedCmPerSecond >= 75.0f) + { + bSpeedAwareTravelMode = true; + } + else if (bSpeedAwareTravelMode && SchedulerSpeedCmPerSecond <= 35.0f) + { + bSpeedAwareTravelMode = false; + } + if (!bSpeedAwareFastMode && SchedulerSpeedCmPerSecond >= 1200.0f) + { + bSpeedAwareFastMode = true; + } + else if (bSpeedAwareFastMode && SchedulerSpeedCmPerSecond <= 800.0f) + { + bSpeedAwareFastMode = false; + } + } + else + { + bSpeedAwareTravelMode = false; + bSpeedAwareFastMode = false; + } + const bool bPawnMoving = SchedulerPolicy == 3 + ? bSpeedAwareTravelMode : SchedulerSpeedCmPerSecond >= 50.0f; bStreamingPlayerMoving = bPawnMoving; // Match the near reserve to the added far queue so queued horizon tiles cannot spend that headroom. // Aligner la reserve proche sur la file lointaine pour que l'horizon ne consomme pas cette marge. @@ -11955,7 +12008,6 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw const int32 NearAdmissionCap = bPawnMoving ? MaxTasks : TotalAdmissionCap; const int32 CoarseTaskCap = FMath::Max(1, MaxTasks - NearTaskReserve) + ReadyQueueDepth; constexpr int32 FirstFarLevel = 3; - const int32 SchedulerPolicy = FMath::Clamp(GVoxelForgeStreamingSchedulerPolicy, 0, 2); const int32 FirstCoarseLevel = FMath::Max( 2, Settings ? Settings->FullResClipLevels : 2); const bool bUseCoarseFirstOrder = GVoxelForgeStreamingCoarseFirst != 0 @@ -12458,10 +12510,15 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw return true; }; - // Policy 0 reconstructs L77's coarse-first admission order for the moving-camera control. - if (SchedulerPolicy == 0) + // Policy 0 reconstructs L77. Policy 3 enters that order only after sustained fast travel. + const bool bPolicy3FastTravel = SchedulerPolicy == 3 && bSpeedAwareFastMode; + if (SchedulerPolicy == 0 || bPolicy3FastTravel) { - if (bUseCoarseFirstOrder) + const bool bPolicy3MayAdmit = !bPolicy3FastTravel + || (bWalkableRingReadyForReserve && AreCriticalTilesReady()); + const bool bCoarseFirstForPolicy = bPolicy3FastTravel + ? bWalkableRingReadyForReserve : bUseCoarseFirstOrder; + if (bPolicy3MayAdmit && bCoarseFirstForPolicy) { for (int32 Level = MaxDesiredLevel; Level >= FirstCoarseLevel; --Level) { @@ -12477,7 +12534,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw true); } } - else + else if (bPolicy3MayAdmit) { SubmitLevel(0, EVoxelTileWorkPriority::LOD0, NearAdmissionCap, false); for (int32 Level = 1; Level <= MaxDesiredLevel; ++Level) @@ -12488,6 +12545,63 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw } } } + else if (SchedulerPolicy == 3) + { + // The collision support ring stays first. At rest, finish LOD0-2 before + // admitting low-priority horizon work. While walking, reserve a speed-scaled + // fraction of available task slots for far parents on every update. + const bool bNearAdmissionReady = AreCriticalTilesReady() + && bWalkableRingReadyForReserve; + if (bNearAdmissionReady) + { + if (!bSpeedAwareTravelMode) + { + if (!AreNearLODTilesReady()) + { + for (int32 Level = 0; Level <= 2 && Level <= MaxDesiredLevel; ++Level) + { + const EVoxelTileWorkPriority Priority = Level == 0 + ? EVoxelTileWorkPriority::LOD0 : EVoxelTileWorkPriority::Horizon; + SubmitLevel(Level, Priority, NearAdmissionCap, true); + } + } + else + { + for (int32 Level = FirstFarLevel; Level <= MaxDesiredLevel; ++Level) + { + SubmitLevel(Level, EVoxelTileWorkPriority::Horizon, + CoarseTaskCap, true); + if (GetGenerationAdmissionCount() >= CoarseTaskCap) break; + } + } + } + else + { + const int32 ShareAdmissionCap = FMath::Max(NearAdmissionCap, CoarseTaskCap); + const int32 StartingAdmissions = GetGenerationAdmissionCount(); + const int32 AvailableAdmissions = FMath::Max( + 0, ShareAdmissionCap - StartingAdmissions); + const double SpeedMetersPerSecond = SchedulerSpeedCmPerSecond / 100.0; + const int32 NearPerFar = FMath::Clamp( + FMath::RoundToInt(6.0 / (1.0 + SpeedMetersPerSecond / 4.0)), 1, 4); + const int32 NearShareAdmissions = + (AvailableAdmissions * NearPerFar) / (NearPerFar + 1); + const int32 NearShareCap = StartingAdmissions + NearShareAdmissions; + for (int32 Level = 0; Level <= 2 && Level <= MaxDesiredLevel; ++Level) + { + const EVoxelTileWorkPriority Priority = Level == 0 + ? EVoxelTileWorkPriority::LOD0 : EVoxelTileWorkPriority::Horizon; + SubmitLevel(Level, Priority, NearShareCap, true); + } + for (int32 Level = MaxDesiredLevel; Level >= FirstFarLevel; --Level) + { + SubmitLevel(Level, EVoxelTileWorkPriority::Horizon, + ShareAdmissionCap, true); + if (GetGenerationAdmissionCount() >= ShareAdmissionCap) break; + } + } + } + } else { const bool bNearAdmissionReady = AreCriticalTilesReady() @@ -12587,16 +12701,35 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw { int32 DesiredByLevel[9] = {}; int32 ResidentByLevel[9] = {}; + int32 CoveredByLevel[9] = {}; int32 PendingByLevel[9] = {}; + int32 TraceMaxDesiredLevel = 0; + for (const FVoxelTileKey& Tile : DesiredSorted) + { + TraceMaxDesiredLevel = FMath::Max(TraceMaxDesiredLevel, Tile.Level); + } for (const FVoxelTileKey& Tile : DesiredSorted) { if (Tile.Level < 0 || Tile.Level > 8) continue; ++DesiredByLevel[Tile.Level]; if (LoadedTiles.Contains(Tile)) ++ResidentByLevel[Tile.Level]; if (PendingTiles.Contains(Tile)) ++PendingByLevel[Tile.Level]; + bool bCovered = LoadedTiles.Contains(Tile); + for (int32 AncestorLevel = Tile.Level + 1; + !bCovered && AncestorLevel <= TraceMaxDesiredLevel && AncestorLevel <= 8; + ++AncestorLevel) + { + const int32 Divisor = VF_ClipLevelScale(AncestorLevel - Tile.Level); + const FVoxelTileKey Ancestor( + VoxelClipmapDesiredTiles::FloorDiv(Tile.Coord, Divisor), + AncestorLevel); + bCovered = LoadedTiles.Contains(Ancestor); + } + if (bCovered) ++CoveredByLevel[Tile.Level]; } FString DesiredCounts = TEXT("["); FString ResidentCounts = TEXT("["); + FString CoveredCounts = TEXT("["); FString PendingCounts = TEXT("["); for (int32 Level = 0; Level <= 8; ++Level) { @@ -12604,14 +12737,17 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw { DesiredCounts += TEXT(","); ResidentCounts += TEXT(","); + CoveredCounts += TEXT(","); PendingCounts += TEXT(","); } DesiredCounts += FString::FromInt(DesiredByLevel[Level]); ResidentCounts += FString::FromInt(ResidentByLevel[Level]); + CoveredCounts += FString::FromInt(CoveredByLevel[Level]); PendingCounts += FString::FromInt(PendingByLevel[Level]); } DesiredCounts += TEXT("]"); ResidentCounts += TEXT("]"); + CoveredCounts += TEXT("]"); PendingCounts += TEXT("]"); const FVector LocalPawn = WorldToLocalCm(CenterPosition); const double StreamingUpdateMilliseconds = (TraceNow - StreamingUpdateStartSeconds) * 1000.0; @@ -12619,6 +12755,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw TEXT("\"scope\":\"desired_residency_not_visual_enclosure\",\"center\":[%d,%d,%d]," "\"position_cm\":[%s,%s,%s],\"speed_cm_s\":%s,\"walkable_ready\":%d," "\"desired_by_lod\":%s,\"resident_by_lod\":%s,\"pending_by_lod\":%s," + "\"covered_by_lod\":%s," "\"active_generation\":%d,\"admission_occupancy\":%d," "\"task_cap\":%d,\"near_reserve\":%d," "\"coarse_first\":%d,\"dynamic_near_reserve\":%d," @@ -12629,7 +12766,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw *FString::SanitizeFloat(LocalPawn.Z), *FString::SanitizeFloat(PlayerHeading.Size()), bWalkableRingReadyForReserve ? 1 : 0, - *DesiredCounts, *ResidentCounts, *PendingCounts, + *DesiredCounts, *ResidentCounts, *PendingCounts, *CoveredCounts, GetActiveGenerationTaskCount(), GetGenerationAdmissionCount(), MaxTasks, NearTaskReserve, GVoxelForgeStreamingCoarseFirst, GVoxelForgeStreamingDynamicNearReserve, GVoxelForgeStreamingLegacyPendingResultCap, diff --git a/Source/VoxelForge/Public/VoxelWorld.h b/Source/VoxelForge/Public/VoxelWorld.h index e41234a..3639a85 100644 --- a/Source/VoxelForge/Public/VoxelWorld.h +++ b/Source/VoxelForge/Public/VoxelWorld.h @@ -1240,6 +1240,8 @@ public: bool bStartupApplyConfigTraced = false; uint64 LastApplyFrameCycles = 0; float StartupTraceSpeedCmPerSecond = 0.0f; + bool bSpeedAwareTravelMode = false; + bool bSpeedAwareFastMode = false; TArray DesiredSorted; // critical floor prefix, then true-pawn distance-first TArray CriticalDesiredTiles; // occupied/support tile, then heading tile // Ancestor -> desired descendants, rebuilt with DesiredSorted. Reconciliation uses this @@ -1347,6 +1349,7 @@ public: double HeadlessStreamingTestDistanceCm = 0.0; float HeadlessStreamingTestSpeedCmPerSecond = 0.0f; float HeadlessStreamingTestStartSeconds = 0.0f; + float HeadlessStreamingTestMoveDurationSeconds = 0.0f; float HeadlessStreamingTestExitSeconds = 0.0f; float HeadlessStreamingTestTeleportAtSeconds = 0.0f; float HeadlessStreamingTestTeleportDistanceCm = 0.0f; diff --git a/Standalone/Research/WorldAssemblerM3/Linux80/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux80/REPORT.md new file mode 100644 index 0000000..d2b5664 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux80/REPORT.md @@ -0,0 +1,73 @@ +# Linux80 + +## Scheduler policy + +Policy 3 is speed aware. The walkable LOD0 ring always leads. On arrival and at rest, LOD0–2 run before far work; far generation continues after the near queue drains. While walking, near and far tiles interleave at about 4:1, with near tiles ordered toward travel. At fast speed, far coarse tiles follow the walkable ring. Headless and Unreal use matching hysteresis: walking enters at 0.75 m/s and exits at 0.35 m/s; fast travel enters at 12 m/s and exits at 8 m/s. + +Headless moving runs used 12 s of motion, 5 s of dwell, 69 samples per run, and had zero generation failures. Values are moving means; dwell is final far holes / near readiness. + +| Speed | Policy | Far holes | Near ready | Dwell end | +|---|---|---:|---:|---:| +| 1.4 m/s | L77 | 7.70% | 75.74% | 0% / 100% | +| 1.4 m/s | L79 | 32.84% | 82.08% | 0% / 100% | +| 1.4 m/s | L80 | 26.62% | 81.54% | 0% / 100% | +| 30 m/s | L77 | 8.96% | 49.40% | 0% / 100% | +| 30 m/s | L79 | 77.21% | 56.96% | 0% / 100% | +| 30 m/s | L80 | 8.94% | 46.50% | 0% / 100% | + +Unreal null-RHI runs used the same 12 s + 5 s route, with no address-space cap. Values are moving means; each dwell ended with 0% far holes and 99.7–99.9% near readiness. + +| Speed | Policy | Far holes | Near ready | Commandlet | +|---|---|---:|---:|---| +| 1.4 m/s | L77 | 5.46% | 99.12% | pass | +| 1.4 m/s | L79 | 7.49% | 99.08% | pass | +| 1.4 m/s | L80 | 9.23% | 99.13% | pass | +| 30 m/s | L77 | 24.89% | 20.79% | exit 1* | +| 30 m/s | L79 | 13.42% | 20.41% | exit 1* | +| 30 m/s | L80 | 10.51% | 20.52% | exit 1* | + +*All fast traces completed their movement and dwell samples, but the commandlet readiness check exited 1: desired=2,237, loaded=2,233, pending/cooks/active=0. Their trace data is usable for the table; the initial-readiness check remains a failed run. L80’s first walking attempt similarly exited with 2,235/2,237 loaded; its complete rerun passed. + +## Cold arrival and timing + +Seconds to LOD0–2 ready / whole view ready; three measured runs per policy and stratum, reported as medians. + +| Stratum | L77 | L79 | L80 (HEAD) | +|---|---:|---:|---:| +| S0 | 2.798 / 2.798 | 1.488 / 2.730 | 1.298 / 2.917 | +| S1 | 4.577 / 4.577 | 3.672 / 4.904 | 3.607 / 5.119 | +| S2 | 4.501 / 4.501 | 3.691 / 4.720 | 3.068 / 4.318 | +| S3 | 2.624 / 2.624 | 1.579 / 2.451 | 1.429 / 2.476 | + +Protocol: rebuilt clean clang headless binary, 8 workers, identical data/flags, no external sampler; one unrecorded warm-up per stratum and policy, then three interleaved triplets in rotating policy order. Every headless process used `taskset -c 0-11`, `prlimit --as=6442450944`, and a 1,740 s timeout. L77/L79 are control policy IDs in the same final binary. L78’s 5.26 s and its L79 re-measure at 3.39 s are not comparable to these timings; neither used this binary and protocol. The older moving-camera values are likewise context only; this report reruns all three policies under one binary. + +## Passage-cut audit + +The L77 census profiled 1,602 tile visits. I selected the highest measured LOD0/1 segment-activity tiles across all four strates: 17 stratum records (14 unique tile keys), covering mouths, route body and support columns. L77’s census attributes 63% of near-LOD FI/TN CPU to passage work. On these active records, a zero-padding bound projected skips of 50.76% of passage iterations, 61.50% of carve visits, 49.76% of support visits and 51.72% of relief calls. That justified testing it. + +The zero-padding candidate changed these samples: + +| Stratum | Changed samples | Sign flips | Max absolute density delta location | Max surface displacement location | +|---|---:|---:|---|---| +| S0 | 19,434 | 0 | 54.329 at tile (-1,-2,-24), LOD1, sample (-34,-98,-1538) | 0 cm; no displaced crossing edge | +| S1 | 63 | 0 | 2.198 at tile (-1,1,-43), LOD1, sample (-2,118,-2748) | 1.014 cm at edge start (-2,116,-2748), axis 1 | +| S2 | 86 | 0 | 2.619 at tile (-1,1,-45), LOD1, sample (0,114,-2854) | 6.641 cm at edge start (0,112,-2854), axis 1 | +| S3 | 21 | 0 | 0.455 at tile (1,-1,-55), LOD1, sample (80,-2,-3484) | 0 cm; no displaced crossing edge | + +Per-sample base/candidate density, delta and sign-flip rows are in `Saved/Linux/l80/passage-census/deltas/strate{0..3}-candidate.csv`; the per-tile logs give individual counts and surface-edge details. The matching `*-candidate-pad48.csv` files contain headers only because there were no changed samples. + +Padding to 48 voxels made all selected output samples identical: zero changes, zero sign flips and 0 cm displacement in every stratum. It also reduced projected skips to 0.226% of iterations, 0.098% of carve visits, 0.196% of support visits and 0.138% of relief calls; paired tile timings were mixed, with no consistent speedup. The useful cut therefore loses passage influence; the safe cut does not save meaningful CPU. The candidate was removed, with no field/version change, captures or peer-determinism run. + +## Build, failures and files + +The final Linux host build succeeded after timing with `-MaxParallelActions=4` using UBT’s local executor. An earlier UBA attempt made no action progress and was stopped before 30 minutes. The first local build caught an out-of-scope `MaxDesiredLevel` reference in the new Unreal coverage trace; that was fixed before the successful build. A first UBT attempt also failed before compilation when it tried to rotate a log in the read-only user config directory; the project config was temporarily redirected and restored. + +Changed `Source/` files: + +- `Source/VoxelForge/Private/VoxelWorld.cpp` — Unreal scheduler, hysteresis, movement test and coverage trace. +- `Source/VoxelForge/Public/VoxelWorld.h` — movement-test duration state. +- `Standalone/Source/StreamingWorld.h` — headless speed-aware order and matching hysteresis. +- `Standalone/Source/StreamBenchmarkHeadless.cpp` — forwards route speed to the scheduler. +- `Standalone/Source/StreamBenchmarkPipeline.h` — surface-delta location reporting. + +Logs and runner scripts are under `Saved/Linux/l80/` and this directory. No commit, push or stash was made. diff --git a/Standalone/Research/WorldAssemblerM3/Linux80/RunL80HeadlessMovingCamera.py b/Standalone/Research/WorldAssemblerM3/Linux80/RunL80HeadlessMovingCamera.py new file mode 100644 index 0000000..69f9ba9 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux80/RunL80HeadlessMovingCamera.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Run the L77/L79/L80 headless moving-camera and dwell protocol.""" +from __future__ import annotations +import csv +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l80/movement/headless" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l80/bin/VoxelForgeCoreHeadless-l80-clean-clang" +CAMERA = (19.166015625, -14.2001953125, -753.3740234375) +POLICIES = {0: "L77", 2: "L79", 3: "L80"} +SPEEDS = {"walk": 1.4, "fast": 30.0} +SAMPLE = re.compile(r"^\[L79MovingSample\] (.*)$", re.M) +SUMMARY = re.compile(r"^\[L79MovingSummary\] (.*)$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") + + +def run(policy: int, speed_name: str) -> tuple[dict, list[dict]]: + label = f"{speed_name}-{POLICIES[policy]}" + log = OUT / f"{label}.log" + OUT.mkdir(parents=True, exist_ok=True) + camera = ",".join(f"{v:.12f}" for v in CAMERA) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", + "timeout", "1740s", str(BINARY), str(DATA), "--world-assembler-prototype", + "--stream-benchmark", "--stream-pipeline", "--workers=8", "--clip-radius=3", + "--max-clip-level=8", "--max-level=8", f"--camera={camera}", + "--profile-lod-work", f"--stream-scheduler-policy={policy}", + f"--moving-camera-speed-mps={SPEEDS[speed_name]:.1f}", + "--moving-camera-duration-s=12", "--moving-camera-dwell-s=5", + ] + start = time.monotonic() + with log.open("w", encoding="utf-8") as output: + result = subprocess.run(command, cwd=ROOT, stdout=output, + stderr=subprocess.STDOUT, check=False, timeout=1740) + content = log.read_text(encoding="utf-8", errors="replace") + samples = [dict(FIELDS.findall(match.group(1))) for match in SAMPLE.finditer(content)] + summary_match = SUMMARY.search(content) + engine = dict(FIELDS.findall(summary_match.group(1))) if summary_match else {} + if result.returncode or not samples or not engine: + raise SystemExit(f"moving-camera run failed or trace missing: {log}; exit={result.returncode}") + for row in samples: + for name in ("policy", "speed_mps", "t_s", "near_ready_pct", "far_missing_pct", + "far_hole_pct", "far_stale_max_s", "generation_failures"): + row[name] = float(row[name]) + row["label"] = label + if int(float(engine.get("generation_failures", "-1"))) != 0: + raise SystemExit(f"tile generation failure: {log}") + run_info = { + "label": label, "policy": POLICIES[policy], "policy_id": policy, + "speed": speed_name, "speed_mps": SPEEDS[speed_name], + "exit_code": result.returncode, "wall_s": round(time.monotonic() - start, 3), + "samples": len(samples), "engine_summary": engine, + "log": str(log.relative_to(ROOT)), + } + print("[L80HeadlessMoveRun] " + json.dumps(run_info, sort_keys=True), flush=True) + return run_info, samples + + +def main() -> int: + if not DATA.is_file() or not BINARY.is_file(): + raise SystemExit(f"missing input: {DATA} or {BINARY}") + runs, all_samples = [], [] + schedule = [(policy, speed) for speed in ("walk", "fast") + for policy in (0, 2, 3)] + for policy, speed in schedule: + row, samples = run(policy, speed) + runs.append(row) + all_samples.extend(samples) + (OUT / "runs-partial.json").write_text( + json.dumps(runs, indent=2) + "\n", encoding="utf-8") + summary = [] + for speed_name, speed in SPEEDS.items(): + for policy in POLICIES.values(): + rows = [row for row in all_samples + if row["speed_mps"] == speed and int(row["policy"]) == + next(pid for pid, name in POLICIES.items() if name == policy)] + moving = [row for row in rows if row.get("phase") == "move"] + dwell = [row for row in rows if row.get("phase") == "dwell"] + summary.append({ + "speed": speed_name, "speed_mps": speed, "policy": policy, + "moving_samples": len(moving), "dwell_samples": len(dwell), + "far_holes_moving_mean_pct": round(sum(r["far_hole_pct"] for r in moving) + / max(1, len(moving)), 3), + "far_holes_moving_max_pct": max((r["far_hole_pct"] for r in moving), default=None), + "near_ready_moving_mean_pct": round(sum(r["near_ready_pct"] for r in moving) + / max(1, len(moving)), 3), + "near_ready_moving_min_pct": min((r["near_ready_pct"] for r in moving), default=None), + "far_holes_dwell_last_pct": dwell[-1]["far_hole_pct"] if dwell else None, + "near_ready_dwell_last_pct": dwell[-1]["near_ready_pct"] if dwell else None, + "first_all_far_ready_s": next( + (r["engine_summary"].get("first_all_far_ready_s") for r in runs + if r["speed"] == speed_name and r["policy"] == policy), None), + "generation_failures": next( + (r["engine_summary"].get("generation_failures") for r in runs + if r["speed"] == speed_name and r["policy"] == policy), None), + }) + (OUT / "summary.json").write_text( + json.dumps({"runs": runs, "summary": summary}, indent=2) + "\n", + encoding="utf-8") + sample_path = OUT / "samples.csv" + if all_samples: + with sample_path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=list(all_samples[0])) + writer.writeheader() + writer.writerows(all_samples) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux80/RunL80PassageCensus.py b/Standalone/Research/WorldAssemblerM3/Linux80/RunL80PassageCensus.py new file mode 100644 index 0000000..84e1988 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux80/RunL80PassageCensus.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Find active LOD0/1 passage tiles in each owner stratum.""" +from __future__ import annotations +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l80/passage-census" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l79/bin/VoxelForgeCoreHeadless-l79-final" +CAMERAS = { + 0: (8.0, 16.0, -372.2626953125), + 1: (25.8369140625, 3.19921875, -737.0), + 2: (19.166015625, -14.2001953125, -753.3740234375), + 3: (-20.0625, -5.068359375, -900.353515625), +} +TILE = re.compile(r"^\[PassageComposerTile\] (.*)$", re.M) +FIELD = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") +KEY = re.compile(r"key=\((-?\d+),(-?\d+),(-?\d+),lod(\d+)\)") + + +def run(strate: int, camera: tuple[float, float, float]) -> dict: + log = OUT / f"strate{strate}.log" + camera_arg = ",".join(f"{v:.12f}" for v in camera) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", + "timeout", "1740s", str(BINARY), str(DATA), "--world-assembler-prototype", + "--stream-benchmark", "--stream-pipeline", "--workers=8", "--clip-radius=3", + "--max-clip-level=8", "--max-level=1", f"--camera={camera_arg}", + "--profile-lod-work", "--profile-passage-composer", + ] + start = time.monotonic() + with log.open("w", encoding="utf-8") as out: + proc = subprocess.run(command, cwd=ROOT, stdout=out, stderr=subprocess.STDOUT, + check=False, timeout=1740) + text = log.read_text(encoding="utf-8", errors="replace") + rows = [] + for match in TILE.finditer(text): + row = dict(FIELD.findall(match.group(1))) + for name, value in tuple(row.items()): + try: + row[name] = int(value) if name not in {"worker_ms"} else float(value) + except ValueError: + pass + key = KEY.search(match.group(1)) + if key: + row["coord"] = [int(key.group(i)) for i in range(1, 4)] + row["lod"] = int(key.group(4)) + row["strate"] = strate + rows.append(row) + summary = next((line for line in text.splitlines() + if line.startswith("[StreamPipeline] ")), None) + result = {"strate": strate, "exit_code": proc.returncode, + "wall_s": round(time.monotonic() - start, 3), + "profiled_tiles": len(rows), "log": str(log.relative_to(ROOT)), + "summary": summary, "tiles": rows} + print("[L80PassageCensus] " + json.dumps( + {k: v for k, v in result.items() if k != "tiles"}, sort_keys=True), flush=True) + if proc.returncode or not rows: + raise SystemExit(f"census failed or emitted no tile rows: {log} exit={proc.returncode}") + return result + + +def main() -> int: + OUT.mkdir(parents=True, exist_ok=True) + runs = [run(i, CAMERAS[i]) for i in range(4)] + tiles = [tile for result in runs for tile in result["tiles"]] + categories = { + "mouth": "mouth_route_segment_checks", + "route_body": "carve_segment_visits", + "support": "support_segment_visits", + } + leaders = {} + for strate in range(4): + for lod in (0, 1): + for category, metric in categories.items(): + candidates = [tile for tile in tiles + if tile["strate"] == strate and tile["lod"] == lod + and tile.get(metric, 0) > 0] + candidates.sort(key=lambda x: (x.get(metric, 0), x.get("worker_ms", 0)), + reverse=True) + leaders[f"s{strate}-l{lod}-{category}"] = candidates[:2] + payload = {"runs": [{k: v for k, v in r.items() if k != "tiles"} for r in runs], + "profiled_tile_count": len(tiles), "tiles": tiles, "leaders": leaders} + (OUT / "summary.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux80/RunL80TimingPairs.py b/Standalone/Research/WorldAssemblerM3/Linux80/RunL80TimingPairs.py new file mode 100644 index 0000000..692c962 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux80/RunL80TimingPairs.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Clean, sampler-free, interleaved cold-readiness comparison for L77/L79/L80.""" +from __future__ import annotations +import json +import re +import statistics +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l80/timing" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l80/bin/VoxelForgeCoreHeadless-l80-clean-clang" +POLICIES = {0: "L77", 2: "L79", 3: "L80"} +CAMERAS = { + 0: (8.0, 16.0, -372.2626953125), + 1: (25.8369140625, 3.19921875, -737.0), + 2: (19.166015625, -14.2001953125, -753.3740234375), + 3: (-20.0625, -5.068359375, -900.353515625), +} +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") +NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) + + +def run(strate: int, policy: int, phase: str, replicate: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + label = f"s{strate}-{POLICIES[policy]}-{phase}{replicate}" + log = OUT / f"{label}.log" + camera = ",".join(f"{v:.12f}" for v in CAMERAS[strate]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", + "timeout", "1740s", str(BINARY), str(DATA), "--world-assembler-prototype", + "--stream-benchmark", "--stream-pipeline", "--workers=8", "--clip-radius=3", + "--max-clip-level=8", "--max-level=8", f"--camera={camera}", + "--profile-lod-work", f"--stream-scheduler-policy={policy}", + ] + start = time.monotonic() + with log.open("w", encoding="utf-8") as output: + result = subprocess.run(command, cwd=ROOT, stdout=output, + stderr=subprocess.STDOUT, check=False, timeout=1740) + content = log.read_text(encoding="utf-8", errors="replace") + pipeline_match = PIPELINE.search(content) + near_match = NEAR.search(content) + if result.returncode or pipeline_match is None or near_match is None: + raise SystemExit(f"readiness run failed or trace missing: {log}; exit={result.returncode}") + pipeline = dict(FIELDS.findall(pipeline_match.group(1))) + row = { + "strate": strate, "policy": POLICIES[policy], "policy_id": policy, + "phase": phase, "replicate": replicate, "exit_code": result.returncode, + "wall_s": round(time.monotonic() - start, 3), + "lod02_ready_s": float(near_match.group(3)), + "whole_view_ready_s": float(pipeline["whole_view_ready"].removesuffix("s")), + "whole_tiles": int(pipeline["whole_tiles"].split("/")[0]), + "log": str(log.relative_to(ROOT)), + } + print("[L80TimingRun] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + if not DATA.is_file() or not BINARY.is_file(): + raise SystemExit(f"missing input: {DATA} or {BINARY}") + rows = [] + # One unrecorded warm-up per stratum and policy. It warms binary/data pages only. + for strate in range(4): + for policy in (0, 2, 3): + run(strate, policy, "warmup", 0) + # Three interleaved triplets; rotate order to keep any drift from favoring one policy. + orders = ((0, 2, 3), (3, 2, 0), (2, 0, 3)) + for strate in range(4): + for replicate, order in enumerate(orders, 1): + for policy in order: + row = run(strate, policy, "measured", replicate) + rows.append(row) + (OUT / "runs-partial.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + summary = [] + for strate in range(4): + group = [row for row in rows if row["strate"] == strate] + entry = {"strate": strate, "measured_pairs_per_policy": 3} + for policy in ("L77", "L79", "L80"): + subset = [row for row in group if row["policy"] == policy] + entry[f"{policy}_lod02_s"] = [row["lod02_ready_s"] for row in subset] + entry[f"{policy}_lod02_median_s"] = statistics.median( + row["lod02_ready_s"] for row in subset) + entry[f"{policy}_whole_s"] = [row["whole_view_ready_s"] for row in subset] + entry[f"{policy}_whole_median_s"] = statistics.median( + row["whole_view_ready_s"] for row in subset) + summary.append(entry) + (OUT / "summary.json").write_text( + json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n", + encoding="utf-8") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux80/RunL80UnrealMovingCamera.py b/Standalone/Research/WorldAssemblerM3/Linux80/RunL80UnrealMovingCamera.py new file mode 100644 index 0000000..5899940 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux80/RunL80UnrealMovingCamera.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Run the Unreal null-RHI moving and dwell trace for L77/L79/L80.""" +from __future__ import annotations +import csv +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +HOST = Path("/home/jahni/VFHost") +HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l80" +HOST_SAVED = HOST / "Saved/Linux/l80" +EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd") +OUT = ROOT / "Saved/Linux/l80/movement/unreal" +SPAWN_M = (19.166015625, -14.2001953125, -753.3740234375) +SPEEDS = {"walk": 140.0, "fast": 3000.0} +POLICIES = {0: "L77", 2: "L79", 3: "L80"} +SAMPLE_SECONDS = 0.25 +MOVE_START_S = 6.0 +MOVE_DURATION_S = 12.0 +DWELL_S = 5.0 +EXIT_S = MOVE_START_S + MOVE_DURATION_S + DWELL_S +EVENT = re.compile(r"^\{" ) + + +def parse_trace(path: Path, run: dict) -> tuple[list[dict], dict]: + events = [] + if path.is_file(): + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if EVENT.match(line): + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + pass + samples = [e for e in events if e.get("type") == "streaming_sample"] + move = [e for e in samples if float(e.get("speed_cm_s", 0.0)) >= 50.0] + stop = next((e for e in events if e.get("type") == "streaming_test_move_stopped"), None) + stop_t = float(stop.get("t_s", 0.0)) if stop else ( + float(move[0].get("t_s", 0.0)) + MOVE_DURATION_S if move else float("inf")) + dwell = [e for e in samples if float(e.get("t_s", 0.0)) >= stop_t + and float(e.get("speed_cm_s", 0.0)) < 50.0] + rows = [] + for e in samples: + desired = e.get("desired_by_lod", [0] * 9) + resident = e.get("resident_by_lod", [0] * 9) + covered = e.get("covered_by_lod", resident) + far_desired = sum(desired[3:]) + far_covered = sum(covered[3:]) + rows.append({ + "run": run["label"], "policy": run["policy"], "speed": run["speed"], + "t_s": e.get("t_s"), "speed_cm_s": e.get("speed_cm_s"), + "far_desired": far_desired, "far_covered": far_covered, + "far_hole_pct": round(100.0 * (far_desired - far_covered) + / max(1, far_desired), 3), + "near_desired": sum(desired[:3]), "near_resident": sum(resident[:3]), + "near_ready_pct": round(100.0 * sum(resident[:3]) + / max(1, sum(desired[:3])), 3), + "phase": "move" if float(e.get("speed_cm_s", 0.0)) >= 50.0 else + "dwell" if float(e.get("t_s", 0.0)) >= stop_t else "arrival", + }) + moving_rows = [r for r in rows if r["phase"] == "move"] + dwell_rows = [r for r in rows if r["phase"] == "dwell"] + summary = { + "moving_samples": len(moving_rows), "dwell_samples": len(dwell_rows), + "far_holes_moving_mean_pct": round(sum(r["far_hole_pct"] for r in moving_rows) + / max(1, len(moving_rows)), 3), + "near_ready_moving_mean_pct": round(sum(r["near_ready_pct"] for r in moving_rows) + / max(1, len(moving_rows)), 3), + "far_holes_dwell_last_pct": dwell_rows[-1]["far_hole_pct"] if dwell_rows else None, + "near_ready_dwell_last_pct": dwell_rows[-1]["near_ready_pct"] if dwell_rows else None, + "first_all_far_ready_s": next((r["t_s"] for r in moving_rows + if r["far_hole_pct"] == 0.0), None), + "move_stopped_t_s": stop_t if stop else None, + } + return rows, summary + + +def run(policy: int, speed_name: str) -> tuple[dict, list[dict]]: + label = f"{speed_name}-{POLICIES[policy]}" + trace, abslog, stdout = (OUT / f"{label}.trace.jsonl", OUT / f"{label}.log", + OUT / f"{label}.stdout.log") + output = HOST_PLUGIN_SAVED / "output" / label + userdir = HOST_SAVED / "user" / label + zendata = HOST_SAVED / "zen" / label + ddc = HOST_SAVED / "DDC" + for directory in (OUT, output, userdir, zendata, ddc): + directory.mkdir(parents=True, exist_ok=True) + spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M) + command = [ + str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", + "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", + "-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer", + "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", + f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", + "-archetype=TunnelNetwork", "-slot=0", "-modes=export", "-gameconfig=1", + "-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25", + "-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0", + "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0", + f"-out={output}", "-voxel.StreamingCoarseFirst=1", + "-voxel.StreamingDynamicNearReserve=1", "-voxel.StreamingLegacyPendingResultCap=0", + "-voxel.CoreRegionConcurrentAdmission=1", "-voxel.StreamingResultBacklogCap=256", + "-voxel.StreamingReadyQueueDepth=8", "-voxel.StreamingBackgroundWorkerCap=1", + "-voxel.CoreColumnCacheDirectMapped=0", "-voxel.CoreMCGradientMemoization=1", + "-voxel.StreamingCollisionLookahead=1", "-voxel.TestMove=1", + f"-voxel.TestMoveSpeedCmPerSecond={SPEEDS[speed_name]:.1f}", + f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}", + f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}", + f"-voxel.TestExitSeconds={EXIT_S:.1f}", "-voxel.TestMoveDirection=1,0,0", + f"-voxel.StreamingSchedulerPolicy={policy}", "-voxel.TestExitOnSteady=0", + "-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}", + "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", + "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", + f"-abslog={abslog}", + ] + wrapped = ["taskset", "-c", "0-11", "timeout", "1740s", *command] + start = time.monotonic() + with stdout.open("w", encoding="utf-8") as stream: + result = subprocess.run(wrapped, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + run_info = {"label": label, "policy": POLICIES[policy], "policy_id": policy, + "speed": speed_name, "speed_cm_s": SPEEDS[speed_name], + "exit_code": result.returncode, "wall_s": round(time.monotonic()-start, 3), + "trace": str(trace.relative_to(ROOT)), "abslog": str(abslog.relative_to(ROOT)), + "stdout": str(stdout.relative_to(ROOT))} + rows, summary = parse_trace(trace, run_info) + run_info.update(summary) + stdout_text = stdout.read_text(encoding="utf-8", errors="replace") + failure_match = re.search( + r"\[VoxelForgeCoreWorldTest\] failed: ([^\r\n]+)", stdout_text) + if result.returncode: + run_info["run_failure"] = (failure_match.group(1) if failure_match + else f"process exit {result.returncode}") + if (not rows or summary["moving_samples"] < 40 or summary["dwell_samples"] < 16): + raise SystemExit(f"Unreal trace incomplete: {stdout}; exit={result.returncode}; " + f"moving={summary['moving_samples']} dwell={summary['dwell_samples']}") + print("[L80UnrealMoveRun] " + json.dumps(run_info, sort_keys=True), flush=True) + return run_info, rows + + +def main() -> int: + if not EDITOR.is_file(): + raise SystemExit(f"missing Unreal commandlet executable: {EDITOR}") + results, samples = [], [] + for speed in ("walk", "fast"): + for policy in (0, 2, 3): + run_info, rows = run(policy, speed) + results.append(run_info) + samples.extend(rows) + (OUT / "runs-partial.json").write_text(json.dumps(results, indent=2) + "\n", + encoding="utf-8") + (OUT / "summary.json").write_text(json.dumps({"runs": results}, indent=2) + "\n", + encoding="utf-8") + if samples: + with (OUT / "samples.csv").open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=list(samples[0])) + writer.writeheader() + writer.writerows(samples) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Source/StreamBenchmarkHeadless.cpp b/Standalone/Source/StreamBenchmarkHeadless.cpp index db37659..71e2a7f 100644 --- a/Standalone/Source/StreamBenchmarkHeadless.cpp +++ b/Standalone/Source/StreamBenchmarkHeadless.cpp @@ -1414,7 +1414,7 @@ namespace std::printf("Usage: %s [owner_world.json] --world-assembler-prototype --stream-benchmark --stream-pipeline [options]\n" " --workers=N --clip-radius=N --max-clip-level=N --max-level=N\n" " --camera=X,Y,Z,YAW,PITCH (position in metres) --stream-hash\n" - " --stream-scheduler-policy=0|1|2 (L77 coarse, L78 near, L79 fair share)\n" + " --stream-scheduler-policy=0|1|2|3 (L77 coarse, L78 near, L79 fair, L80 speed-aware)\n" " --moving-camera-speed-mps=N [--moving-camera-duration-s=N] [--moving-camera-dwell-s=N]\n" " --stream-hash-tiles (print per-tile hashes; implies --stream-hash)\n" " --stream-hash-tiles-allow-skips (mesh/topology falsifier with proven uniform skips)\n" @@ -1600,8 +1600,8 @@ namespace else if(arg.starts_with("--stream-scheduler-policy=")) { if(!readInt("--stream-scheduler-policy=",options.StreamingSchedulerPolicy) - ||options.StreamingSchedulerPolicy<0||options.StreamingSchedulerPolicy>2) - {std::fprintf(stderr,"Invalid --stream-scheduler-policy (expected 0..2).\n");return false;} + ||options.StreamingSchedulerPolicy<0||options.StreamingSchedulerPolicy>3) + {std::fprintf(stderr,"Invalid --stream-scheduler-policy (expected 0..3).\n");return false;} } else if(arg.starts_with("--moving-camera-speed-mps=")) { @@ -1676,7 +1676,7 @@ namespace if(options.MovingCameraTrace&&(!options.StreamPipeline||!options.WorldAssemblerPrototype)) {std::fprintf(stderr,"--moving-camera-speed-mps requires --world-assembler-prototype --stream-pipeline\n");return false;} if(options.MovingCameraTrace&&options.StreamingSchedulerPolicy<0) - {std::fprintf(stderr,"--moving-camera-speed-mps requires --stream-scheduler-policy=0|1|2\n");return false;} + {std::fprintf(stderr,"--moving-camera-speed-mps requires --stream-scheduler-policy=0|1|2|3\n");return false;} if(options.LandmarkL45RampControl&&!options.LandmarkChamberDemo) {std::fprintf(stderr,"--landmark-l45-ramp-control requires --landmark-chamber-demo\n");return false;} if(options.AuditOreVeinYield&&!options.OreVeinsDemo) diff --git a/Standalone/Source/StreamBenchmarkPipeline.h b/Standalone/Source/StreamBenchmarkPipeline.h index bc2f745..5083df0 100644 --- a/Standalone/Source/StreamBenchmarkPipeline.h +++ b/Standalone/Source/StreamBenchmarkPipeline.h @@ -38,6 +38,8 @@ namespace StandaloneStreamBenchmark double sumAbsoluteDelta=0.0; double maxSurfaceDisplacementVoxels=0.0; double sumSurfaceDisplacementVoxels=0.0; + std::array maxSurfaceDisplacementStartVoxel{}; + std::int32_t maxSurfaceDisplacementAxis=-1; bool valid=false; }; @@ -94,7 +96,7 @@ namespace StandaloneStreamBenchmark } const auto CompareEdge=[&](std::int32_t X0,std::int32_t Y0,std::int32_t Z0, - std::int32_t X1,std::int32_t Y1,std::int32_t Z1) + std::int32_t X1,std::int32_t Y1,std::int32_t Z1,std::int32_t Axis) { const bool CrossA=EdgeCrosses(A.debugDensitySamples,X0,Y0,Z0,X1,Y1,Z1); const bool CrossB=EdgeCrosses(B.debugDensitySamples,X0,Y0,Z0,X1,Y1,Z1); @@ -107,8 +109,13 @@ namespace StandaloneStreamBenchmark const double TA=static_cast(A0)/(static_cast(A0)-A1); const double TB=static_cast(B0)/(static_cast(B0)-B1); const double Displacement=std::abs(TA-TB)*Step; - Result.maxSurfaceDisplacementVoxels=std::max( - Result.maxSurfaceDisplacementVoxels,Displacement); + if(Displacement>Result.maxSurfaceDisplacementVoxels) + { + Result.maxSurfaceDisplacementVoxels=Displacement; + Result.maxSurfaceDisplacementStartVoxel={ + Origin.X+(X0-1)*Step,Origin.Y+(Y0-1)*Step,Origin.Z+(Z0-1)*Step}; + Result.maxSurfaceDisplacementAxis=Axis; + } Result.sumSurfaceDisplacementVoxels+=Displacement; ++Result.sharedCrossingEdges; }; @@ -116,9 +123,9 @@ namespace StandaloneStreamBenchmark for(std::int32_t Y=0;Y(MovingCameraSpeedMetersPerSecond*MovingSeconds/VoxelMetres); + Streamer.SetSchedulerMotionSpeedMetersPerSecond( + ElapsedSeconds=12.0)NextMode=2; + else if(ClampedSpeed>=0.75)NextMode=1; + } + else if(CurrentMode==1) + { + if(ClampedSpeed>=12.0)NextMode=2; + else if(ClampedSpeed<=0.35)NextMode=0; + } + else if(ClampedSpeed<=8.0) + { + NextMode=ClampedSpeed<=0.35?0:1; + } + if(NextMode==CurrentMode + ||SpeedAwareSchedulerMode.compare_exchange_weak( + CurrentMode,NextMode,std::memory_order_relaxed)) + { + break; + } + } + SchedulerMotionSpeedMetersPerSecond.store(ClampedSpeed,std::memory_order_relaxed); + } bool QueuePriorityTasks(std::vector> Tasks) { @@ -800,8 +830,7 @@ private: for(const auto& Pair:CancelledByCrossing)Stats.CancelledByCrossing.push_back(Pair); std::vector Prioritized=NewDesired.Tiles; - const int32 ActiveSchedulerPolicy=SchedulerPolicy<0 - ?(IsCoarseFirstSchedulingEnabled()?2:3):SchedulerPolicy; + const int32 ActiveSchedulerPolicy=SchedulerPolicy<0?3:SchedulerPolicy; if(ActiveSchedulerPolicy<=2) { // Policy 0 retains L77 coarse-first ordering after the local support ring. @@ -865,6 +894,84 @@ private: Prioritized.swap(FairOrder); } } + else if(ActiveSchedulerPolicy==3) + { + std::vector WalkableRing,NearTiles,FarTiles,SpeedAwareOrder; + WalkableRing.reserve(27); + NearTiles.reserve(Prioritized.size()); + FarTiles.reserve(Prioritized.size()); + SpeedAwareOrder.reserve(Prioritized.size()); + for(const FVoxelTileKey& Key:Prioritized) + { + const bool bInWalkableRing=Key.Level==0 + &&std::abs(Key.Coord.X-NewDesired.CenterChunk.X)<=1 + &&std::abs(Key.Coord.Y-NewDesired.CenterChunk.Y)<=1 + &&std::abs(Key.Coord.Z-NewDesired.CenterChunk.Z)<=1; + if(bInWalkableRing)WalkableRing.push_back(Key); + else if(Key.Level<=2)NearTiles.push_back(Key); + else FarTiles.push_back(Key); + } + const auto IsAhead=[&](const FVoxelTileKey& Key) + { + if(TravelDirection.X*TravelDirection.X+TravelDirection.Y*TravelDirection.Y<=1.0e-12f) + return false; + const double Scale=static_cast(1<(Key.Coord.X)+0.5)*Scale-NewDesired.CenterChunk.X; + const double OffsetY=(static_cast(Key.Coord.Y)+0.5)*Scale-NewDesired.CenterChunk.Y; + return OffsetX*TravelDirection.X+OffsetY*TravelDirection.Y>0.0; + }; + auto SortGroup=[&](std::vector& Group,bool bNearFirst) + { + std::sort(Group.begin(),Group.end(),[&](const FVoxelTileKey& A,const FVoxelTileKey& B) + { + if(A.Level!=B.Level)return bNearFirst?A.LevelB.Level; + const bool bAheadA=IsAhead(A),bAheadB=IsAhead(B); + if(bAheadA!=bAheadB)return bAheadA; + const double DA=StandaloneWorldMesher::TileDistanceSquaredChunks(A,NewDesired.CenterChunk); + const double DB=StandaloneWorldMesher::TileDistanceSquaredChunks(B,NewDesired.CenterChunk); + if(DA!=DB)return DA(std::clamp( + static_cast(std::lround(6.0/(1.0+SpeedMetersPerSecond/4.0))),1,4)); + size_t NearIndex=0,FarIndex=0; + while(NearIndex Unordered;Unordered.reserve(NewDesired.Tiles.size()); @@ -1084,8 +1191,11 @@ private: std::unique_lock Lock(Mutex); const auto HasEligibleTile=[&]() { - return !Queue.empty()&&(SchedulerPolicy!=1||NearOutstandingJobs==0 - ||Queue.front().Key.Level<=2); + const bool bStationaryNearGate=SchedulerPolicy==3 + &&SpeedAwareSchedulerMode.load(std::memory_order_relaxed)==0 + &&NearOutstandingJobs>0; + return !Queue.empty()&&((SchedulerPolicy!=1&&!bStationaryNearGate) + ||NearOutstandingJobs==0||Queue.front().Key.Level<=2); }; Wake.wait(Lock,[&](){return bStop||!PriorityTasks.empty()||HasEligibleTile();}); if(bStop)return; @@ -1250,6 +1360,9 @@ private: bool bHasLastCameraPosition=false; int32 WorkerCount=8; int32 SchedulerPolicy=-1; + std::atomic SchedulerMotionSpeedMetersPerSecond{0.0}; + // Match Unreal's Schmitt thresholds to keep noisy speed samples from flapping modes. + std::atomic SpeedAwareSchedulerMode{0}; bool bHashDensity=false; const FStandaloneMeshCache* MeshCache=nullptr; int32 MaxLevelFilter=-1;