diff --git a/Source/VoxelForge/Private/VoxelWorld.cpp b/Source/VoxelForge/Private/VoxelWorld.cpp index 42ac6f5..5c436ee 100644 --- a/Source/VoxelForge/Private/VoxelWorld.cpp +++ b/Source/VoxelForge/Private/VoxelWorld.cpp @@ -195,10 +195,13 @@ public: std::atomic bShuttingDown{false}; std::atomic NextEditSequence{1}; std::atomic RegionCompileStartCycles{0}; + std::atomic RegionCompileWorkerStartCycles{0}; std::atomic RegionCompileEndCycles{0}; std::atomic RegionCompileTileCount{0}; std::atomic RegionCompileSerial{0}; + std::atomic RegionCompileWorkerThreadId{0}; std::atomic RegionCompileWorkerSeconds{0.0}; + std::atomic RegionCompileWallSeconds{0.0}; int32 FullResolutionLodCount = 0; FString InitialFailureReason; }; @@ -11381,20 +11384,25 @@ void AVoxelWorld::StartPendingCoreRegionCompile() 1, std::memory_order_relaxed) + 1; State->RegionCompileStartCycles.store(ActiveCoreRegionCompileStartCycles, std::memory_order_release); + State->RegionCompileWorkerStartCycles.store(0, std::memory_order_release); State->RegionCompileEndCycles.store(0, std::memory_order_release); State->RegionCompileTileCount.store(0, std::memory_order_release); + State->RegionCompileWorkerThreadId.store(0, std::memory_order_release); State->RegionCompileWorkerSeconds.store(0.0, std::memory_order_release); + State->RegionCompileWallSeconds.store(0.0, std::memory_order_release); if (VoxelForgeStartupTrace::IsActive()) { VoxelForgeStartupTrace::RecordEvent(TEXT("core_region_compile_start"), FString::Printf( TEXT("\"compile_serial\":%d,\"request_serial\":%d,\"center\":[%d,%d,%d]," "\"missing_region_tiles\":%d,\"retained_tiles\":%d,\"concurrent_admission\":%d," - "\"active_tile_tasks\":%d,\"active_task_guard\":%d"), + "\"active_tile_tasks\":%d,\"active_generation_workers\":%d," + "\"active_task_guard\":%d,\"game_thread_id\":%u"), CompileSerial, CoreRegionCompileRequestSerial, CompileCenter.X, CompileCenter.Y, CompileCenter.Z, MissingTiles.Num(), RetainedTiles.Num(), bAllowConcurrentGeneration ? 1 : 0, ActiveTileTaskCount.load(std::memory_order_relaxed), - ActiveTaskCount.load(std::memory_order_relaxed))); + ActiveGenerationWorkerCount.load(std::memory_order_relaxed), + ActiveTaskCount.load(std::memory_order_relaxed), FPlatformTLS::GetCurrentThreadId())); } ActiveTaskCount.fetch_add(1, std::memory_order_relaxed); UE::Tasks::Launch(TEXT("VoxelForgeCoreTileRegion"), @@ -11407,6 +11415,11 @@ void AVoxelWorld::StartPendingCoreRegionCompile() ~FTaskGuard() { Counter.fetch_sub(1, std::memory_order_release); } } Guard{ActiveTaskCount}; + State->RegionCompileWorkerStartCycles.store(FPlatformTime::Cycles64(), + std::memory_order_release); + State->RegionCompileWorkerThreadId.store(FPlatformTLS::GetCurrentThreadId(), + std::memory_order_release); + if (bShuttingDown.load(std::memory_order_acquire) || State->bShuttingDown.load(std::memory_order_acquire)) { @@ -11458,6 +11471,8 @@ void AVoxelWorld::StartPendingCoreRegionCompile() } State->RegionCompileWorkerSeconds.store(CompileStats.workerSeconds, std::memory_order_release); + State->RegionCompileWallSeconds.store(CompileStats.wallSeconds, + std::memory_order_release); State->bRegionCompileSucceeded.store(bCompiled, std::memory_order_release); if (!bCompiled) { @@ -12345,9 +12360,17 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw State->RegionCompileStartCycles.load(std::memory_order_acquire); const uint64 CompileEndCycles = State->RegionCompileEndCycles.load(std::memory_order_acquire); + const uint64 CompileWorkerStartCycles = + State->RegionCompileWorkerStartCycles.load(std::memory_order_acquire); const double CompileElapsedMilliseconds = CompileEndCycles >= CompileStartCycles ? FPlatformTime::ToSeconds64(CompileEndCycles - CompileStartCycles) * 1000.0 : 0.0; + const double CompileQueueMilliseconds = CompileWorkerStartCycles >= CompileStartCycles + ? FPlatformTime::ToSeconds64(CompileWorkerStartCycles - CompileStartCycles) * 1000.0 + : 0.0; + const double CompileTaskMilliseconds = CompileEndCycles >= CompileWorkerStartCycles + ? FPlatformTime::ToSeconds64(CompileEndCycles - CompileWorkerStartCycles) * 1000.0 + : 0.0; const double GateElapsedMilliseconds = CoreRegionPendingSinceSeconds > 0.0 ? FMath::Max(0.0, FPlatformTime::Seconds() - CoreRegionPendingSinceSeconds) * 1000.0 : 0.0; @@ -12355,6 +12378,10 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw State->RegionCompileTileCount.load(std::memory_order_acquire); const double CompileWorkerSeconds = State->RegionCompileWorkerSeconds.load(std::memory_order_acquire); + const double CompileCoreWallSeconds = + State->RegionCompileWallSeconds.load(std::memory_order_acquire); + const uint32 CompileWorkerThreadId = + State->RegionCompileWorkerThreadId.load(std::memory_order_acquire); const int32 CompileSerial = State->RegionCompileSerial.load(std::memory_order_acquire); const uint64 IndependentSubmissions = CoreRegionIndependentSubmissions; @@ -12364,14 +12391,20 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw VoxelForgeStartupTrace::RecordEvent(TEXT("core_region_compile_complete"), FString::Printf( TEXT("\"compile_serial\":%d,\"request_serial\":%d,\"center\":[%d,%d,%d]," "\"success\":%d,\"compiled_tiles\":%d,\"compile_elapsed_ms\":%s," - "\"compile_worker_s\":%s,\"pending_gate_elapsed_ms\":%s," + "\"compile_queue_ms\":%s,\"compile_task_ms\":%s," + "\"compile_core_wall_ms\":%s,\"compile_worker_s\":%s," + "\"compile_worker_thread_id\":%u,\"pending_gate_elapsed_ms\":%s," "\"independent_submissions\":%llu," "\"dependent_declines\":%llu,\"concurrent_admission\":%d"), CompileSerial, ActiveCoreRegionCompileRequestSerial, ActiveCoreRegionCompileCenter.X, ActiveCoreRegionCompileCenter.Y, ActiveCoreRegionCompileCenter.Z, bRegionCompileSucceeded ? 1 : 0, CompiledTileCount, *FString::SanitizeFloat(CompileElapsedMilliseconds), + *FString::SanitizeFloat(CompileQueueMilliseconds), + *FString::SanitizeFloat(CompileTaskMilliseconds), + *FString::SanitizeFloat(CompileCoreWallSeconds * 1000.0), *FString::SanitizeFloat(CompileWorkerSeconds), + CompileWorkerThreadId, *FString::SanitizeFloat(GateElapsedMilliseconds), static_cast(IndependentSubmissions), static_cast(DependentDeclines), @@ -12584,6 +12617,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw && bWalkableRingReadyForReserve; int32 SchedulerSubmittedByLevel[9] = {}; int32 SchedulerLoadAttemptsByLevel[9] = {}; + int32 SchedulerRegionCompileDeclinesByLevel[9] = {}; + int32 SchedulerFarParentDeclinesByLevel[9] = {}; int32 SchedulerLevelCapReturns[9] = {}; int32 SchedulerLevelCapBreaks[9] = {}; int32 SchedulerCriticalFullCapBreaks = 0; @@ -12951,7 +12986,21 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw if(CoreState.IsValid() &&CoreState->bFarWorldRequired.load(std::memory_order_acquire) &&!CoreState->bFarWorldReady.load(std::memory_order_acquire)) + { + if (bTraceStreamingScheduler && Tile.Level >= 0 && Tile.Level < 9) + { + ++SchedulerFarParentDeclinesByLevel[Tile.Level]; + } return; + } + } + if (bTraceStreamingScheduler && bUseVoxelForgeCore && bCoreRegionCompilePending + && GVoxelForgeCoreRegionConcurrentAdmission != 0 + && (PendingCoreRegionTileSet.Contains(Tile) + || CoreRegionCompileTilesInFlight.Contains(Tile)) + && Tile.Level >= 0 && Tile.Level < 9) + { + ++SchedulerRegionCompileDeclinesByLevel[Tile.Level]; } if (Tile.Level >= 0 && Tile.Level < 9) { @@ -13661,10 +13710,37 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw const double BusySeconds = FPlatformTime::ToSeconds64(WorkerCyclesDelta); const double SchedulerUpdateMilliseconds = (SchedulerTraceNow - StreamingUpdateStartSeconds) * 1000.0; + int32 CriticalMissingTiles = 0; + for (const FVoxelTileKey& Tile : CriticalDesiredTiles) + { + if (!LoadedTiles.Contains(Tile) && !CollisionNotRequiredTiles.Contains(Tile)) + { + ++CriticalMissingTiles; + } + } + int32 CollisionGateMissingTiles = 0; + for (const FVoxelTileKey& Tile : CollisionGateDesiredTiles) + { + if (!LoadedTiles.Contains(Tile) && !CollisionNotRequiredTiles.Contains(Tile)) + { + ++CollisionGateMissingTiles; + } + } + const FIntVector DesiredCenterLag = DesiredEpochCenterChunk - CenterChunk; + const TSharedPtr TraceCoreState = + VoxelForgeCoreState; + const bool bFarWorldRequiredForTrace = TraceCoreState.IsValid() + && TraceCoreState->bFarWorldRequired.load(std::memory_order_relaxed); + const bool bFarWorldReadyForTrace = TraceCoreState.IsValid() + && TraceCoreState->bFarWorldReady.load(std::memory_order_relaxed); VoxelForgeStartupTrace::RecordEvent(TEXT("scheduler_admission"), FString::Printf( TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"desired_rebuilt\":%d," "\"submission_loop_ran\":%d,\"all_chunks_loaded\":%d," "\"walkable_ready\":%d,\"moving\":%d," + "\"pawn_collision_gate_engaged\":%d,\"critical_tiles_ready\":%d," + "\"critical_missing\":%d,\"collision_gate_missing\":%d," + "\"desired_center_lag\":[%d,%d,%d],\"desired_center_current\":%d," + "\"far_world_required\":%d,\"far_world_ready\":%d," "\"core_region_compile_pending\":%d,\"core_ready\":%d,\"coarse_first\":%d," "\"dynamic_near_reserve\":%d,\"legacy_pending_result_cap\":%d," "\"core_region_concurrent_admission\":%d," @@ -13685,6 +13761,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw "\"level_candidate_scan_ms\":%s,\"passage_score_ms\":%s," "\"level_sort_ms\":%s,\"level_admit_ms\":%s," "\"desired_missing_by_lod\":%s,\"loaded_by_lod\":%s,\"pending_by_lod\":%s," + "\"region_compile_declines_by_lod\":%s," + "\"far_parent_declines_by_lod\":%s," "\"submitted_by_lod\":%s,\"load_attempts_by_lod\":%s," "\"admission_cap_by_lod\":%s,\"declined_rule_by_lod\":%s," "\"level_cap_returns_by_lod\":%s,\"level_cap_breaks_by_lod\":%s," @@ -13695,6 +13773,11 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw static_cast(GFrameCounter), CenterChunk.X, CenterChunk.Y, CenterChunk.Z, bDesiredRebuiltThisUpdate ? 1 : 0, bSubmissionLoopRan ? 1 : 0, bAllChunksLoaded ? 1 : 0, bWalkableRingReadyForReserve ? 1 : 0, bPawnMoving ? 1 : 0, + bPawnCollisionGateEngaged ? 1 : 0, CriticalMissingTiles == 0 ? 1 : 0, + CriticalMissingTiles, CollisionGateMissingTiles, + DesiredCenterLag.X, DesiredCenterLag.Y, DesiredCenterLag.Z, + DesiredCenterLag == FIntVector::ZeroValue ? 1 : 0, + bFarWorldRequiredForTrace ? 1 : 0, bFarWorldReadyForTrace ? 1 : 0, bCoreRegionCompilePending ? 1 : 0, IsVoxelForgeCoreReady() ? 1 : 0, GVoxelForgeStreamingCoarseFirst, GVoxelForgeStreamingDynamicNearReserve, GVoxelForgeStreamingLegacyPendingResultCap, @@ -13721,7 +13804,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw *FString::SanitizeFloat(LevelSortMilliseconds), *FString::SanitizeFloat(LevelAdmissionMilliseconds), *IntArrayJson(DesiredMissingByLevel), *IntArrayJson(LoadedByLevel), - *IntArrayJson(PendingByLevel), *IntArrayJson(SchedulerSubmittedByLevel), + *IntArrayJson(PendingByLevel), *IntArrayJson(SchedulerRegionCompileDeclinesByLevel), + *IntArrayJson(SchedulerFarParentDeclinesByLevel), *IntArrayJson(SchedulerSubmittedByLevel), *IntArrayJson(SchedulerLoadAttemptsByLevel), *IntArrayJson(AdmissionCapByLevel), *IntArrayJson(DeclinedRuleByLevel), *IntArrayJson(SchedulerLevelCapReturns), *IntArrayJson(SchedulerLevelCapBreaks), FullCapDeclinedTiles, ReserveDeclinedTiles,