streaming: near LODs first (walkable ring, LOD0, LOD1, LOD2, then far); exact passage trims (L78)

- Headless and Unreal schedulers drain the collision-critical ring, then LOD0-2, then far LODs
  at low priority. LOD0-2 ready (W8 medians): SW 4.36->1.76, FI 6.89->5.26, TN 4.88->3.22,
  VS 2.71->1.56 s. Whole view roughly unchanged (FI/VS slightly later).
  KNOWN RISK: far tiles wait until every near tile is ready, so far LODs may starve while
  moving fast (to address next).
- Exact composer trims: tile-level upper-mouth candidate list, segment-index min/max rejection
  before lower_bound, 64-entry thread-local value-noise corner cache. Passage share 65 -> 63%.
- World exact: 24/24 hash matrix + pinned L75 owner hash. Fast tier PASS. Linux UE host builds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed 2026-10-27 19:13:45 +01:00
1 parent 5e091d254c
commit ec20994a8e
10 files changed
+718 -52

No files matched your search

+67 -39
View File
@@ -206,7 +206,7 @@ namespace
FAutoConsoleVariableRef CVarVoxelForgeStreamingCoarseFirst( FAutoConsoleVariableRef CVarVoxelForgeStreamingCoarseFirst(
TEXT("voxel.StreamingCoarseFirst"), TEXT("voxel.StreamingCoarseFirst"),
GVoxelForgeStreamingCoarseFirst, GVoxelForgeStreamingCoarseFirst,
TEXT("Prioritize current walkable tiles, then far coarse clipmap levels, then near refinement; 0 restores legacy LOD order.")); TEXT("Prioritize walkable tiles, then finish LOD0-2 before far clipmap work; controls far-level order."));
int32 GVoxelForgeStreamingDynamicNearReserve = 1; int32 GVoxelForgeStreamingDynamicNearReserve = 1;
FAutoConsoleVariableRef CVarVoxelForgeStreamingDynamicNearReserve( FAutoConsoleVariableRef CVarVoxelForgeStreamingDynamicNearReserve(
@@ -11631,7 +11631,8 @@ void AVoxelWorld::EnqueueStreamingResult(FChunkResult&& Result)
{ {
EditedProcessQueue.Enqueue(MoveTemp(Result)); EditedProcessQueue.Enqueue(MoveTemp(Result));
} }
else if (Result.WorkPriority == EVoxelTileWorkPriority::LOD0) else if (Result.WorkPriority == EVoxelTileWorkPriority::LOD0
|| Result.Tile.Level <= 2)
{ {
NearProcessQueue.Enqueue(MoveTemp(Result)); NearProcessQueue.Enqueue(MoveTemp(Result));
} }
@@ -11947,8 +11948,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
const int32 NearTaskReserve = EffectiveNearTaskReserve; const int32 NearTaskReserve = EffectiveNearTaskReserve;
const int32 NearAdmissionCap = bPawnMoving ? MaxTasks : TotalAdmissionCap; const int32 NearAdmissionCap = bPawnMoving ? MaxTasks : TotalAdmissionCap;
const int32 CoarseTaskCap = FMath::Max(1, MaxTasks - NearTaskReserve) + ReadyQueueDepth; const int32 CoarseTaskCap = FMath::Max(1, MaxTasks - NearTaskReserve) + ReadyQueueDepth;
const int32 FirstCoarseLevel = FMath::Max( constexpr int32 FirstFarLevel = 3;
2, Settings ? Settings->FullResClipLevels : 2);
const bool bUseCoarseFirstOrder = GVoxelForgeStreamingCoarseFirst != 0 const bool bUseCoarseFirstOrder = GVoxelForgeStreamingCoarseFirst != 0
&& bWalkableRingReadyForReserve; && bWalkableRingReadyForReserve;
int32 SchedulerSubmittedByLevel[9] = {}; int32 SchedulerSubmittedByLevel[9] = {};
@@ -12425,43 +12425,65 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
} }
}; };
// Coarse-first gives the far shell the first non-critical admissions. Keep the current const auto AreCriticalTilesReady = [this]()
// collision-ready 3x3 ring above it, then spend the remaining slots on full-resolution
// refinement. The opt-out reproduces the S4 LOD0-to-LOD8 submission order for a control.
// Le shell lointain passe apres l'anneau de collision, puis le raffinement fin. L'option
// inverse restaure l'ordre LOD0-vers-LOD8 de S4 comme controle negatif.
// Keep the walkable ring on the legacy near-first path; only spend its reserved task slots
// on coarse work after all nine collision gates are ready.
// Garder l'anneau praticable en priorite ; les taches grossieres commencent apres ses neuf collisions.
if (bUseCoarseFirstOrder)
{ {
for (int32 Level = MaxDesiredLevel; Level >= FirstCoarseLevel; --Level) for (const FVoxelTileKey& Tile : CriticalDesiredTiles)
{ {
SubmitLevel(Level, EVoxelTileWorkPriority::Horizon, if (!LoadedTiles.Contains(Tile) && !CollisionNotRequiredTiles.Contains(Tile))
CoarseTaskCap, true); {
return false;
}
} }
for (int32 Level = 0; Level < FirstCoarseLevel && Level <= MaxDesiredLevel; ++Level) return true;
};
const auto AreNearLODTilesReady = [this]()
{
for (const FVoxelTileKey& Tile : DesiredSorted)
{
if (Tile.Level <= 2 && !LoadedTiles.Contains(Tile)
&& !CollisionNotRequiredTiles.Contains(Tile))
{
return false;
}
}
return true;
};
// The support and collision-critical set drains first. Then LOD0, LOD1, and LOD2
// use the near lane. Far tiles are not admitted until every requested near tile is ready.
if (AreCriticalTilesReady() && bWalkableRingReadyForReserve)
{
for (int32 Level = 0; Level <= 2 && Level <= MaxDesiredLevel; ++Level)
{ {
const EVoxelTileWorkPriority Priority = Level == 0 const EVoxelTileWorkPriority Priority = Level == 0
? EVoxelTileWorkPriority::LOD0 : EVoxelTileWorkPriority::Horizon; ? EVoxelTileWorkPriority::LOD0 : EVoxelTileWorkPriority::Horizon;
const int32 LevelAdmissionCap = Priority == EVoxelTileWorkPriority::Horizon SubmitLevel(Level, Priority, NearAdmissionCap, true);
? CoarseTaskCap : NearAdmissionCap;
SubmitLevel(Level, Priority, LevelAdmissionCap, true);
} }
} }
else
if (AreNearLODTilesReady())
{ {
SubmitLevel(0, EVoxelTileWorkPriority::LOD0, NearAdmissionCap, false); if (bUseCoarseFirstOrder)
for (int32 Level = 1; Level <= MaxDesiredLevel; ++Level)
{ {
SubmitLevel(Level, EVoxelTileWorkPriority::Horizon, for (int32 Level = MaxDesiredLevel; Level >= FirstFarLevel; --Level)
CoarseTaskCap, false);
if (GetGenerationAdmissionCount() >= CoarseTaskCap)
{ {
if (Level >= 0 && Level < 9) { ++SchedulerLevelCapBreaks[Level]; } SubmitLevel(Level, EVoxelTileWorkPriority::Horizon,
SchedulerOrderStopLevel = Level; CoarseTaskCap, true);
SchedulerOrderStopCap = CoarseTaskCap; }
break; }
else
{
for (int32 Level = FirstFarLevel; Level <= MaxDesiredLevel; ++Level)
{
SubmitLevel(Level, EVoxelTileWorkPriority::Horizon,
CoarseTaskCap, false);
if (GetGenerationAdmissionCount() >= CoarseTaskCap)
{
if (Level >= 0 && Level < 9) { ++SchedulerLevelCapBreaks[Level]; }
SchedulerOrderStopLevel = Level;
SchedulerOrderStopCap = CoarseTaskCap;
break;
}
} }
} }
} }
@@ -12621,9 +12643,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
> ResultBacklogCapStopsAtUpdateEntry; > ResultBacklogCapStopsAtUpdateEntry;
for (int32 Level = 0; Level < 9; ++Level) for (int32 Level = 0; Level < 9; ++Level)
{ {
AdmissionCapByLevel[Level] = bUseCoarseFirstOrder AdmissionCapByLevel[Level] = Level <= 2 ? NearAdmissionCap : CoarseTaskCap;
? (Level >= FirstCoarseLevel || Level > 0 ? CoarseTaskCap : NearAdmissionCap)
: (Level == 0 ? NearAdmissionCap : CoarseTaskCap);
if (DesiredMissingByLevel[Level] <= 0) continue; if (DesiredMissingByLevel[Level] <= 0) continue;
if (bAllChunksLoaded) if (bAllChunksLoaded)
@@ -12711,7 +12731,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
"\"max_tasks\":%d,\"ready_queue_depth\":%d," "\"max_tasks\":%d,\"ready_queue_depth\":%d,"
"\"total_admission_cap\":%d,\"near_admission_cap\":%d," "\"total_admission_cap\":%d,\"near_admission_cap\":%d,"
"\"near_reserve\":%d," "\"near_reserve\":%d,"
"\"coarse_cap\":%d,\"first_coarse_level\":%d," "\"coarse_cap\":%d,\"first_far_level\":%d,"
"\"submission_stop_level\":%d,\"submission_stop_cap\":%d," "\"submission_stop_level\":%d,\"submission_stop_cap\":%d,"
"\"pending_entry\":%d,\"pending_after\":%d,\"admission_occupancy\":%d," "\"pending_entry\":%d,\"pending_after\":%d,\"admission_occupancy\":%d,"
"\"active_task_guard\":%d,\"running_workers\":%d,\"pending_results_estimate\":%d," "\"active_task_guard\":%d,\"running_workers\":%d,\"pending_results_estimate\":%d,"
@@ -12734,7 +12754,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
GVoxelForgeStreamingCoarseFirst, GVoxelForgeStreamingDynamicNearReserve, GVoxelForgeStreamingCoarseFirst, GVoxelForgeStreamingDynamicNearReserve,
GVoxelForgeStreamingLegacyPendingResultCap, GVoxelForgeStreamingLegacyPendingResultCap,
GVoxelForgeCoreRegionConcurrentAdmission, MaxTasks, ReadyQueueDepth, GVoxelForgeCoreRegionConcurrentAdmission, MaxTasks, ReadyQueueDepth,
TotalAdmissionCap, NearAdmissionCap, NearTaskReserve, CoarseTaskCap, FirstCoarseLevel, TotalAdmissionCap, NearAdmissionCap, NearTaskReserve, CoarseTaskCap, FirstFarLevel,
SchedulerOrderStopLevel, SchedulerOrderStopCap, SchedulerOrderStopLevel, SchedulerOrderStopCap,
PendingTilesAtUpdateEntry, PendingTiles.Num(), AdmissionOccupancy, ActiveTaskGuard, PendingTilesAtUpdateEntry, PendingTiles.Num(), AdmissionOccupancy, ActiveTaskGuard,
ActiveGenerationWorkerCount.load(std::memory_order_relaxed), PendingResultsEstimate, ActiveGenerationWorkerCount.load(std::memory_order_relaxed), PendingResultsEstimate,
@@ -12789,15 +12809,19 @@ void AVoxelWorld::LoadTile(const FVoxelTileKey& Tile, EVoxelTileWorkPriority Wor
const int32 ReadyQueueDepth = FMath::Clamp( const int32 ReadyQueueDepth = FMath::Clamp(
GVoxelForgeStreamingReadyQueueDepth, 0, FMath::Min(64, FMath::Max(1, MaxTasks) * 4)); GVoxelForgeStreamingReadyQueueDepth, 0, FMath::Min(64, FMath::Max(1, MaxTasks) * 4));
const int32 TotalAdmissionCap = MaxTasks + ReadyQueueDepth; const int32 TotalAdmissionCap = MaxTasks + ReadyQueueDepth;
const bool bNearPriority = WorkPriority != EVoxelTileWorkPriority::Horizon; const bool bNearPriority = WorkPriority != EVoxelTileWorkPriority::Horizon
|| Tile.Level <= 2;
const int32 NearReserve = EffectiveNearTaskReserve >= 0 const int32 NearReserve = EffectiveNearTaskReserve >= 0
? FMath::Clamp(EffectiveNearTaskReserve, 0, FMath::Max(0, MaxTasks - 1)) ? FMath::Clamp(EffectiveNearTaskReserve, 0, FMath::Max(0, MaxTasks - 1))
: GetNearTaskReserve(MaxTasks); : GetNearTaskReserve(MaxTasks);
const int32 NearAdmissionCap = bStreamingPlayerMoving ? MaxTasks : TotalAdmissionCap; const int32 NearAdmissionCap = bStreamingPlayerMoving ? MaxTasks : TotalAdmissionCap;
const int32 CoarseTaskCap = FMath::Max(1, MaxTasks - NearReserve) + ReadyQueueDepth; const int32 CoarseTaskCap = FMath::Max(1, MaxTasks - NearReserve) + ReadyQueueDepth;
const int32 AdmissionCap = WorkPriority == EVoxelTileWorkPriority::Horizon const int32 AdmissionCap = WorkPriority == EVoxelTileWorkPriority::Horizon
&& Tile.Level > 2
? CoarseTaskCap ? CoarseTaskCap
: (WorkPriority == EVoxelTileWorkPriority::LOD0 ? NearAdmissionCap : TotalAdmissionCap); : ((WorkPriority == EVoxelTileWorkPriority::LOD0
|| (WorkPriority == EVoxelTileWorkPriority::Horizon && Tile.Level <= 2))
? NearAdmissionCap : TotalAdmissionCap);
if (GetGenerationAdmissionCount() >= AdmissionCap) if (GetGenerationAdmissionCount() >= AdmissionCap)
{ {
if (GVoxelForgeProfileTileGeneration != 0 if (GVoxelForgeProfileTileGeneration != 0
@@ -13132,15 +13156,19 @@ void AVoxelWorld::LoadCoreTile(const FVoxelTileKey& Tile, EVoxelTileWorkPriority
const int32 ReadyQueueDepth = FMath::Clamp( const int32 ReadyQueueDepth = FMath::Clamp(
GVoxelForgeStreamingReadyQueueDepth, 0, FMath::Min(64, FMath::Max(1, MaxTasks) * 4)); GVoxelForgeStreamingReadyQueueDepth, 0, FMath::Min(64, FMath::Max(1, MaxTasks) * 4));
const int32 TotalAdmissionCap = MaxTasks + ReadyQueueDepth; const int32 TotalAdmissionCap = MaxTasks + ReadyQueueDepth;
const bool bNearPriority = WorkPriority != EVoxelTileWorkPriority::Horizon; const bool bNearPriority = WorkPriority != EVoxelTileWorkPriority::Horizon
|| Tile.Level <= 2;
const int32 NearReserve = EffectiveNearTaskReserve >= 0 const int32 NearReserve = EffectiveNearTaskReserve >= 0
? FMath::Clamp(EffectiveNearTaskReserve, 0, FMath::Max(0, MaxTasks - 1)) ? FMath::Clamp(EffectiveNearTaskReserve, 0, FMath::Max(0, MaxTasks - 1))
: GetNearTaskReserve(MaxTasks); : GetNearTaskReserve(MaxTasks);
const int32 NearAdmissionCap = bStreamingPlayerMoving ? MaxTasks : TotalAdmissionCap; const int32 NearAdmissionCap = bStreamingPlayerMoving ? MaxTasks : TotalAdmissionCap;
const int32 CoarseTaskCap = FMath::Max(1, MaxTasks - NearReserve) + ReadyQueueDepth; const int32 CoarseTaskCap = FMath::Max(1, MaxTasks - NearReserve) + ReadyQueueDepth;
const int32 AdmissionCap = WorkPriority == EVoxelTileWorkPriority::Horizon const int32 AdmissionCap = WorkPriority == EVoxelTileWorkPriority::Horizon
&& Tile.Level > 2
? CoarseTaskCap ? CoarseTaskCap
: (WorkPriority == EVoxelTileWorkPriority::LOD0 ? NearAdmissionCap : TotalAdmissionCap); : ((WorkPriority == EVoxelTileWorkPriority::LOD0
|| (WorkPriority == EVoxelTileWorkPriority::Horizon && Tile.Level <= 2))
? NearAdmissionCap : TotalAdmissionCap);
if (GetGenerationAdmissionCount() >= AdmissionCap) return; if (GetGenerationAdmissionCount() >= AdmissionCap) return;
PendingTiles.Add(Tile); PendingTiles.Add(Tile);
+74 -6
View File
@@ -201,6 +201,37 @@ public:
passageColumns_=passageRoutes_; passageColumns_=passageRoutes_;
passageMouthSupports_=passageRoutes_; passageMouthSupports_=passageRoutes_;
passageMouthSupportSegments_.resize(passageRoutes_.size()); passageMouthSupportSegments_.resize(passageRoutes_.size());
if(shape.ResolvedLayout&&!shape.ResolvedLayout->passages.empty())
{
constexpr double Q=static_cast<double>(
VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel);
const double qPerMetre=Q*1000000.0/
std::max<std::uint32_t>(1,
shape.ResolvedLayout->coreVoxelSizeMicrometres);
const double horizontalReach=15.0*qPerMetre;
const double verticalReach=8.0*qPerMetre;
const bool failOpen=minimumSubvoxels_[0]>maximumSubvoxels_[0]
||minimumSubvoxels_[1]>maximumSubvoxels_[1]
||minimumSubvoxels_[2]>maximumSubvoxels_[2];
for(std::size_t passageIndex=0;
passageIndex<shape.ResolvedLayout->passages.size();++passageIndex)
{
if(failOpen)
{
upperMouthSupportPassages_.push_back(passageIndex);
continue;
}
const auto& mouth=shape.ResolvedLayout->passages[
passageIndex].upperMouthSubvoxels;
if(static_cast<double>(mouth[0])+horizontalReach>=minimumSubvoxels_[0]
&&static_cast<double>(mouth[0])-horizontalReach<=maximumSubvoxels_[0]
&&static_cast<double>(mouth[1])+horizontalReach>=minimumSubvoxels_[1]
&&static_cast<double>(mouth[1])-horizontalReach<=maximumSubvoxels_[1]
&&static_cast<double>(mouth[2])+verticalReach>=minimumSubvoxels_[2]
&&static_cast<double>(mouth[2])-verticalReach<=maximumSubvoxels_[2])
upperMouthSupportPassages_.push_back(passageIndex);
}
}
for(const auto& crossing:shape.PassageVoidCrossings) for(const auto& crossing:shape.PassageVoidCrossings)
if(crossing.needsColumn if(crossing.needsColumn
&&crossing.bottomZSubvoxels<=maximumSubvoxels_[2] &&crossing.bottomZSubvoxels<=maximumSubvoxels_[2]
@@ -257,6 +288,10 @@ public:
[](std::uint8_t value){return value!=0;});} [](std::uint8_t value){return value!=0;});}
bool hasRouteFeatureOverlap() const{return routeFeatureOverlap_;} bool hasRouteFeatureOverlap() const{return routeFeatureOverlap_;}
bool hasColumnZOverlap() const{return columnZOverlap_;} bool hasColumnZOverlap() const{return columnZOverlap_;}
bool mayContainUpperMouthSupport() const
{return !upperMouthSupportPassages_.empty();}
const std::vector<std::size_t>& upperMouthSupportPassages() const
{return upperMouthSupportPassages_;}
bool mayContainResolvedBoundaryBand() const{return mayContainResolvedBoundaryBand_;} bool mayContainResolvedBoundaryBand() const{return mayContainResolvedBoundaryBand_;}
bool mayContainTransitionPassageFade() const{return mayContainTransitionPassageFade_;} bool mayContainTransitionPassageFade() const{return mayContainTransitionPassageFade_;}
bool mayContainResolvedBoundaryBandAtZ(double zMicrometres) const bool mayContainResolvedBoundaryBandAtZ(double zMicrometres) const
@@ -282,6 +317,7 @@ public:
std::vector<std::uint32_t> cavernFeatureIndices_; std::vector<std::uint32_t> cavernFeatureIndices_;
std::vector<std::uint8_t> passageRoutes_,passageColumns_,passageMouthSupports_; std::vector<std::uint8_t> passageRoutes_,passageColumns_,passageMouthSupports_;
std::vector<std::vector<FCandidateSegment>> passageMouthSupportSegments_; std::vector<std::vector<FCandidateSegment>> passageMouthSupportSegments_;
std::vector<std::size_t> upperMouthSupportPassages_;
std::size_t routeSegmentCandidateCount_=0; std::size_t routeSegmentCandidateCount_=0;
bool routeFeatureOverlap_=false,columnZOverlap_=false; bool routeFeatureOverlap_=false,columnZOverlap_=false;
bool mayContainResolvedBoundaryBand_=true; bool mayContainResolvedBoundaryBand_=true;
@@ -1004,6 +1040,11 @@ public:
auto* passageProfile=passageTileScopeMatches auto* passageProfile=passageTileScopeMatches
?passageTileQueries->profileStats():nullptr; ?passageTileQueries->profileStats():nullptr;
if(passageProfile)++passageProfile->passageComposerQueries; if(passageProfile)++passageProfile->passageComposerQueries;
if(passageTileScopeMatches&&density>=0.0f
&&!passageTileQueries->hasRouteFeatureOverlap()
&&!passageTileQueries->hasColumnZOverlap()
&&!passageTileQueries->hasAnyMouthSupport()
&&!passageTileQueries->mayContainUpperMouthSupport())return density;
bool queryNearUpperMouthSupport=false; bool queryNearUpperMouthSupport=false;
const auto checkUpperMouth=[&](const auto& passage) const auto checkUpperMouth=[&](const auto& passage)
{ {
@@ -1015,10 +1056,22 @@ public:
if(dx*dx+dy*dy<=reach*reach&&std::abs(dz)<=8.0*qPerMetre) if(dx*dx+dy*dy<=reach*reach&&std::abs(dz)<=8.0*qPerMetre)
queryNearUpperMouthSupport=true; queryNearUpperMouthSupport=true;
}; };
for(const auto& passage:ResolvedLayout->passages) if(passageTileScopeMatches)
{ {
checkUpperMouth(passage); for(const std::size_t passageIndex:
if(queryNearUpperMouthSupport)break; passageTileQueries->upperMouthSupportPassages())
{
checkUpperMouth(ResolvedLayout->passages[passageIndex]);
if(queryNearUpperMouthSupport)break;
}
}
else
{
for(const auto& passage:ResolvedLayout->passages)
{
checkUpperMouth(passage);
if(queryNearUpperMouthSupport)break;
}
} }
if(passageTileScopeMatches&&density>=0.0f if(passageTileScopeMatches&&density>=0.0f
&&!passageTileQueries->hasRouteFeatureOverlap() &&!passageTileQueries->hasRouteFeatureOverlap()
@@ -3862,6 +3915,7 @@ private:
{ {
std::array<double,3> minimum{},maximum{}; std::array<double,3> minimum{},maximum{};
double maximumRadius=0.0; double maximumRadius=0.0;
std::size_t minimumSegment=0,maximumSegment=0;
std::vector<std::size_t> segments; std::vector<std::size_t> segments;
int left=-1,right=-1; int left=-1,right=-1;
}; };
@@ -4676,6 +4730,8 @@ private:
node.maximumRadius=std::max({node.maximumRadius, node.maximumRadius=std::max({node.maximumRadius,
static_cast<double>(a.radiusSubvoxels),static_cast<double>(b.radiusSubvoxels)}); static_cast<double>(a.radiusSubvoxels),static_cast<double>(b.radiusSubvoxels)});
} }
node.minimumSegment=*std::min_element(segments.begin(),segments.end());
node.maximumSegment=*std::max_element(segments.begin(),segments.end());
node.segments=segments; node.segments=segments;
std::sort(node.segments.begin(),node.segments.end()); std::sort(node.segments.begin(),node.segments.end());
const int nodeIndex=static_cast<int>(PassageNearestNodes[passageIndex].size()); const int nodeIndex=static_cast<int>(PassageNearestNodes[passageIndex].size());
@@ -5284,7 +5340,11 @@ private:
if(passageIndex>=PassageNearestNodes.size()||nodeIndex<0)return; if(passageIndex>=PassageNearestNodes.size()||nodeIndex<0)return;
const auto& nodes=PassageNearestNodes[passageIndex]; const auto& nodes=PassageNearestNodes[passageIndex];
const auto& node=nodes[static_cast<std::size_t>(nodeIndex)]; const auto& node=nodes[static_cast<std::size_t>(nodeIndex)];
const auto candidate=std::lower_bound(node.segments.begin(),node.segments.end(),first); if(node.maximumSegment<first||node.minimumSegment>=end)return;
const bool allSegmentsInRange=first<=node.minimumSegment
&&node.maximumSegment<end;
const auto candidate=allSegmentsInRange?node.segments.begin()
:std::lower_bound(node.segments.begin(),node.segments.end(),first);
if(candidate==node.segments.end()||*candidate>=end)return; if(candidate==node.segments.end()||*candidate>=end)return;
const double lowerBound=PassageNearestNodeLowerBound(node,qx,qy,qz); const double lowerBound=PassageNearestNodeLowerBound(node,qx,qy,qz);
if(lowerBound>bestDistance)return; if(lowerBound>bestDistance)return;
@@ -5325,7 +5385,11 @@ private:
if(passageIndex>=PassageNearestNodes.size()||nodeIndex<0)return false; if(passageIndex>=PassageNearestNodes.size()||nodeIndex<0)return false;
const auto& nodes=PassageNearestNodes[passageIndex]; const auto& nodes=PassageNearestNodes[passageIndex];
const auto& node=nodes[static_cast<std::size_t>(nodeIndex)]; const auto& node=nodes[static_cast<std::size_t>(nodeIndex)];
const auto candidate=std::lower_bound(node.segments.begin(),node.segments.end(),first); if(node.maximumSegment<first||node.minimumSegment>=end)return false;
const bool allSegmentsInRange=first<=node.minimumSegment
&&node.maximumSegment<end;
const auto candidate=allSegmentsInRange?node.segments.begin()
:std::lower_bound(node.segments.begin(),node.segments.end(),first);
if(candidate==node.segments.end()||*candidate>=end)return false; if(candidate==node.segments.end()||*candidate>=end)return false;
if(PassageNearestNodeLowerBound(node,qx,qy,qz)>signedDistanceLimit)return false; if(PassageNearestNodeLowerBound(node,qx,qy,qz)>signedDistanceLimit)return false;
if(node.left<0)return true; if(node.left<0)return true;
@@ -5448,7 +5512,11 @@ private:
if(passageIndex>=PassageNearestNodes.size()||nodeIndex<0)return false; if(passageIndex>=PassageNearestNodes.size()||nodeIndex<0)return false;
const auto& nodes=PassageNearestNodes[passageIndex]; const auto& nodes=PassageNearestNodes[passageIndex];
const auto& node=nodes[static_cast<std::size_t>(nodeIndex)]; const auto& node=nodes[static_cast<std::size_t>(nodeIndex)];
auto candidate=std::lower_bound(node.segments.begin(),node.segments.end(),first); if(node.maximumSegment<first||node.minimumSegment>=end)return false;
const bool allSegmentsInRange=first<=node.minimumSegment
&&node.maximumSegment<end;
auto candidate=allSegmentsInRange?node.segments.begin()
:std::lower_bound(node.segments.begin(),node.segments.end(),first);
if(candidate==node.segments.end()||*candidate>=end)return false; if(candidate==node.segments.end()||*candidate>=end)return false;
const double xyReach=0.39*qPerMetre; const double xyReach=0.39*qPerMetre;
if(qx+xyReach<node.minimum[0]||qx-xyReach>node.maximum[0] if(qx+xyReach<node.minimum[0]||qx-xyReach>node.maximum[0]
+11
View File
@@ -1891,6 +1891,17 @@ namespace VoxelForgeCore
std::vector<TileKey> output(tiles.begin(),tiles.end()); std::vector<TileKey> output(tiles.begin(),tiles.end());
std::sort(output.begin(),output.end(),[&](const TileKey& a,const TileKey& b) std::sort(output.begin(),output.end(),[&](const TileKey& a,const TileKey& b)
{ {
const auto tier=[&](const TileKey& key)
{
if(key.lod==0&&std::abs(key.coordinate.x-centerChunk.x)<=1
&&std::abs(key.coordinate.y-centerChunk.y)<=1
&&std::abs(key.coordinate.z-centerChunk.z)<=1)return 0;
return key.lod<=2?1:2;
};
const int tierA=tier(a),tierB=tier(b);
if(tierA!=tierB)return tierA<tierB;
if(tierA==1&&a.lod!=b.lod)return a.lod<b.lod;
if(tierA==2&&a.lod!=b.lod)return a.lod>b.lod;
const double da=TileDistanceSquared(a,centerChunk),db=TileDistanceSquared(b,centerChunk); const double da=TileDistanceSquared(a,centerChunk),db=TileDistanceSquared(b,centerChunk);
if(da!=db)return da<db; if(da!=db)return da<db;
if(a.lod!=b.lod)return a.lod<b.lod; if(a.lod!=b.lod)return a.lod<b.lod;
@@ -20,6 +20,7 @@
#define VF_FIELD_CLAMP(v,a,b) clamp(v,a,b) #define VF_FIELD_CLAMP(v,a,b) clamp(v,a,b)
#define VF_FIELD_LERP(a,b,t) lerp(a,b,t) #define VF_FIELD_LERP(a,b,t) lerp(a,b,t)
#else #else
#include <array>
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
#include <cstdint> #include <cstdint>
@@ -244,11 +245,70 @@ VF_FIELD_INLINE float VF_FieldValueNoise(float x, float y, float z,
return VF_FieldLerp(VF_FieldLerp(z00, z10, ty), VF_FieldLerp(z01, z11, ty), tz); return VF_FieldLerp(VF_FieldLerp(z00, z10, ty), VF_FieldLerp(z01, z11, ty), tz);
} }
#if !defined(VF_FIELD_HLSL)
struct VF_FieldValueNoiseCacheEntry
{
VF_FieldI32 x=0,y=0,z=0,wavelength=0;
VF_FieldU32 seed=0;
bool valid=false;
std::array<float,8> corners{};
};
// Reuse the eight lattice values while adjacent voxel samples remain in the same
// value-noise cell. Only the integer hash lookups are shared; interpolation keeps
// the same inputs and operation order as VF_FieldValueNoise.
VF_FIELD_INLINE float VF_FieldCachedValueNoise(float x,float y,float z,
VF_FieldI32 wavelength,VF_FieldU32 seed)
{
static thread_local std::array<VF_FieldValueNoiseCacheEntry,64> cache{};
const float inv=1.0f/VF_FIELD_CAST_F32(wavelength);
const float fx=x*inv,fy=y*inv,fz=z*inv;
const VF_FieldI32 x0=VF_FIELD_CAST_I32(VF_FIELD_FLOOR(fx));
const VF_FieldI32 y0=VF_FIELD_CAST_I32(VF_FIELD_FLOOR(fy));
const VF_FieldI32 z0=VF_FIELD_CAST_I32(VF_FIELD_FLOOR(fz));
const VF_FieldU32 cellHash=VF_FieldMix(seed
^VF_FIELD_CAST_U32(x0)*0x68E31DA4u
^VF_FIELD_CAST_U32(y0)*0x1B56C4E9u
^VF_FIELD_CAST_U32(z0)*0x9E3779B9u
^VF_FIELD_CAST_U32(wavelength)*0x85EBCA77u);
auto& entry=cache[cellHash&(cache.size()-1)];
if(!entry.valid||entry.x!=x0||entry.y!=y0||entry.z!=z0
||entry.wavelength!=wavelength||entry.seed!=seed)
{
entry.x=x0;entry.y=y0;entry.z=z0;entry.wavelength=wavelength;
entry.seed=seed;entry.valid=true;
entry.corners={
VF_FieldNoiseValue(x0,y0,z0,seed),
VF_FieldNoiseValue(x0+1,y0,z0,seed),
VF_FieldNoiseValue(x0,y0+1,z0,seed),
VF_FieldNoiseValue(x0+1,y0+1,z0,seed),
VF_FieldNoiseValue(x0,y0,z0+1,seed),
VF_FieldNoiseValue(x0+1,y0,z0+1,seed),
VF_FieldNoiseValue(x0,y0+1,z0+1,seed),
VF_FieldNoiseValue(x0+1,y0+1,z0+1,seed)};
}
const float tx=VF_FieldFade(fx-VF_FIELD_CAST_F32(x0));
const float ty=VF_FieldFade(fy-VF_FIELD_CAST_F32(y0));
const float tz=VF_FieldFade(fz-VF_FIELD_CAST_F32(z0));
const float z00=VF_FieldLerp(entry.corners[0],entry.corners[1],tx);
const float z10=VF_FieldLerp(entry.corners[2],entry.corners[3],tx);
const float z01=VF_FieldLerp(entry.corners[4],entry.corners[5],tx);
const float z11=VF_FieldLerp(entry.corners[6],entry.corners[7],tx);
return VF_FieldLerp(VF_FieldLerp(z00,z10,ty),VF_FieldLerp(z01,z11,ty),tz);
}
#endif
VF_FIELD_INLINE float VF_FieldSurfaceRelief(float x, float y, float z, VF_FieldU32 seed) VF_FIELD_INLINE float VF_FieldSurfaceRelief(float x, float y, float z, VF_FieldU32 seed)
{ {
#if defined(VF_FIELD_HLSL)
const float macro = VF_FieldValueNoise(x, y, z, 32, seed ^ 0xB5297A4Du); const float macro = VF_FieldValueNoise(x, y, z, 32, seed ^ 0xB5297A4Du);
const float mid = VF_FieldValueNoise(x, y, z, 16, seed ^ 0x63D83595u); const float mid = VF_FieldValueNoise(x, y, z, 16, seed ^ 0x63D83595u);
const float fine = VF_FieldValueNoise(x, y, z, 4, seed ^ 0xA511E9B3u); const float fine = VF_FieldValueNoise(x, y, z, 4, seed ^ 0xA511E9B3u);
#else
const float macro=VF_FieldCachedValueNoise(x,y,z,32,seed^0xB5297A4Du);
const float mid=VF_FieldCachedValueNoise(x,y,z,16,seed^0x63D83595u);
const float fine=VF_FieldCachedValueNoise(x,y,z,4,seed^0xA511E9B3u);
#endif
return 1.20f * macro + (0.80f + 0.25f * macro) * (0.40f * mid + 0.60f * fine); return 1.20f * macro + (0.80f + 0.25f * macro) * (0.40f * mid + 0.60f * fine);
} }
@@ -335,12 +395,21 @@ VF_FIELD_INLINE float VF_FieldPassageWallRelief(float x,float y,float z,VF_Field
{ {
// Use long, non-commensurate rotated scales. The shared passage wall keeps broad, // Use long, non-commensurate rotated scales. The shared passage wall keeps broad,
// uneven rock relief without a short wavelength that can read as repeated rings. // uneven rock relief without a short wavelength that can read as repeated rings.
#if defined(VF_FIELD_HLSL)
const float broad=VF_FieldValueNoise(x+17.0f,y-23.0f,z+11.0f,43, const float broad=VF_FieldValueNoise(x+17.0f,y-23.0f,z+11.0f,43,
seed^0xB5297A4Du); seed^0xB5297A4Du);
const float mid=VF_FieldValueNoise(y-7.0f,z+19.0f,x-31.0f,29, const float mid=VF_FieldValueNoise(y-7.0f,z+19.0f,x-31.0f,29,
seed^0x63D83595u); seed^0x63D83595u);
const float fine=VF_FieldValueNoise(z+5.0f,x+29.0f,y-13.0f,19, const float fine=VF_FieldValueNoise(z+5.0f,x+29.0f,y-13.0f,19,
seed^0xA511E9B3u); seed^0xA511E9B3u);
#else
const float broad=VF_FieldCachedValueNoise(x+17.0f,y-23.0f,z+11.0f,43,
seed^0xB5297A4Du);
const float mid=VF_FieldCachedValueNoise(y-7.0f,z+19.0f,x-31.0f,29,
seed^0x63D83595u);
const float fine=VF_FieldCachedValueNoise(z+5.0f,x+29.0f,y-13.0f,19,
seed^0xA511E9B3u);
#endif
return 0.50f*broad+0.35f*mid+0.15f*fine; return 0.50f*broad+0.35f*mid+0.15f*fine;
} }
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Summarize sampler closure, phase cost, and route census from L78 timing logs."""
from __future__ import annotations
import csv
import json
import re
import statistics
from collections import defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
TIMING = ROOT / "Saved/Linux/l78/timing"
OUT = ROOT / "Saved/Linux/l78"
SAMPLE = re.compile(
r"^\[L77Sample\] pc=\S+ lod=(\d+) stage=(\S+) route=(\S+) "
r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M)
SAMPLER = re.compile(
r"^\[L77Sampler\] action=stop .*?samples=(\d+) .*?lost=(\d+) "
r"timer_overruns=(\d+).*?profiled_thread_cpu_s=([0-9.]+).*?"
r"handler_cpu_ms=([0-9.]+) handler_share_pct=([0-9.]+) "
r"observed_us_per_tick=([0-9.]+)", re.M)
DISPATCH = re.compile(
r"^\[WorldAssemblerL77Dispatch\] lod=(\d+) stage=(\S+) route=(\S+) "
r"calls=(\d+) points=(\d+)$", re.M)
LOD_COST = re.compile(
r"^\[WorldAssemblerLODCostDetail\] lod=(\d+) tiles=(\d+).*?"
r"mc_grid_ms=([0-9.]+).*?worker_ms_per_tile=([0-9.]+)", re.M)
def analyze(row: dict) -> tuple[dict, list[dict], list[dict]]:
path = ROOT / row["log"]
content = path.read_text(encoding="utf-8", errors="replace")
sampler_match = SAMPLER.search(content)
if sampler_match is None:
raise SystemExit(f"sampler summary missing from {path}")
samples = [
(int(lod), stage, route, phase, int(halo), int(weight))
for lod, stage, route, phase, halo, weight in SAMPLE.findall(content)
]
sample_count, lost, overruns = map(int, sampler_match.group(1, 2, 3))
profiled_cpu_s, handler_cpu_ms, handler_share_pct, period_us = map(
float, sampler_match.group(4, 5, 6, 7))
period_s = period_us / 1_000_000.0
lod_costs = {
int(lod): (int(tiles), float(grid_ms) / 1000.0,
float(worker_ms) * int(tiles) / 1000.0)
for lod, tiles, grid_ms, worker_ms in LOD_COST.findall(content)
}
dispatch = {
(int(lod), stage, route): (int(calls), int(points))
for lod, stage, route, calls, points in DISPATCH.findall(content)
}
scope_metrics = []
for scope, predicate in (("LOD0-2", lambda lod: 0 <= lod <= 2),
("whole_view", lambda lod: 0 <= lod <= 8)):
selected = [item for item in samples if predicate(item[0])]
sampled_cpu_s = sum(item[5] for item in selected) * period_s
sampled_grid_s = sum(item[5] for item in selected if item[1] == "grid_fill") * period_s
sampled_passage_s = sum(item[5] for item in selected
if item[3] == "passages") * period_s
sampled_support_s = sum(item[5] for item in selected
if item[3] == "support") * period_s
direct_grid_s = sum(value[1] for lod, value in lod_costs.items() if predicate(lod))
direct_worker_s = sum(value[2] for lod, value in lod_costs.items() if predicate(lod))
scope_metrics.append({
"strate": row["strate"], "mode": row["mode"], "pair": row["pair"],
"scope": scope, "sample_events": len(selected),
"sampled_cpu_s": sampled_cpu_s,
"sampled_grid_fill_s": sampled_grid_s,
"direct_grid_fill_s": direct_grid_s,
"grid_closure_pct": 100.0 * sampled_grid_s / direct_grid_s if direct_grid_s else 0.0,
"sampled_passage_s": sampled_passage_s,
"sampled_passage_pct": 100.0 * sampled_passage_s / sampled_cpu_s
if sampled_cpu_s else 0.0,
"sampled_support_s": sampled_support_s,
"sampled_support_pct": 100.0 * sampled_support_s / sampled_cpu_s
if sampled_cpu_s else 0.0,
"direct_worker_cpu_s": direct_worker_s,
})
route_metrics = []
scopes = (("LOD0-2", lambda lod: 0 <= lod <= 2),
("whole_view", lambda lod: 0 <= lod <= 8))
for scope, predicate in scopes:
route_keys = {
(item[1], item[2]) for item in samples if predicate(item[0])
} | {
(stage, route) for (lod, stage, route) in dispatch if predicate(lod)
}
sampled_total = sum(item[5] for item in samples if predicate(item[0])) * period_s
for stage, route in sorted(route_keys):
weighted = sum(item[5] for item in samples
if predicate(item[0]) and item[1] == stage and item[2] == route)
calls = sum(value[0] for (lod, current_stage, current_route), value in dispatch.items()
if predicate(lod) and current_stage == stage and current_route == route)
points = sum(value[1] for (lod, current_stage, current_route), value in dispatch.items()
if predicate(lod) and current_stage == stage and current_route == route)
sampled_s = weighted * period_s
route_metrics.append({
"strate": row["strate"], "mode": row["mode"], "pair": row["pair"],
"scope": scope, "caller_stage": stage, "route": route,
"calls": calls, "points": points, "sampled_cpu_s": sampled_s,
"share_scope_sampled_cpu_pct": 100.0 * sampled_s / sampled_total
if sampled_total else 0.0,
})
summary = {
"strate": row["strate"], "mode": row["mode"], "pair": row["pair"],
"log": row["log"], "sampler_events": sample_count,
"parsed_sampler_events": len(samples), "lost": lost,
"timer_overruns": overruns, "profiled_thread_cpu_s": profiled_cpu_s,
"handler_cpu_ms": handler_cpu_ms, "handler_share_pct": handler_share_pct,
"observed_us_per_tick": period_us,
}
return summary, scope_metrics, route_metrics
def write_csv(path: Path, rows: list[dict]) -> None:
if not rows:
return
with path.open("w", encoding="utf-8", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
def main() -> int:
timing = json.loads((TIMING / "summary.json").read_text(encoding="utf-8"))
sampler_rows, scope_rows, route_rows = [], [], []
for row in timing["runs"]:
sampler, scopes, routes = analyze(row)
sampler_rows.append(sampler)
scope_rows.extend(scopes)
route_rows.extend(routes)
write_csv(OUT / "sampler-census.csv", sampler_rows)
write_csv(OUT / "closure-census.csv", scope_rows)
write_csv(OUT / "route-census.csv", route_rows)
averaged = []
for strate in range(4):
for mode in ("before", "after"):
samples = [row for row in scope_rows
if row["strate"] == strate and row["mode"] == mode]
for scope in ("LOD0-2", "whole_view"):
values = [row for row in samples if row["scope"] == scope]
averaged.append({
"strate": strate, "mode": mode, "scope": scope,
"runs": len(values),
"sampled_cpu_s_mean": statistics.mean(row["sampled_cpu_s"] for row in values),
"sampled_grid_fill_s_mean": statistics.mean(row["sampled_grid_fill_s"] for row in values),
"direct_grid_fill_s_mean": statistics.mean(row["direct_grid_fill_s"] for row in values),
"grid_closure_pct_mean": statistics.mean(row["grid_closure_pct"] for row in values),
"passage_s_mean": statistics.mean(row["sampled_passage_s"] for row in values),
"passage_sample_pct_mean": statistics.mean(
row["sampled_passage_pct"] for row in values),
"support_s_mean": statistics.mean(row["sampled_support_s"] for row in values),
"support_sample_pct_mean": statistics.mean(
row["sampled_support_pct"] for row in values),
"direct_worker_cpu_s_mean": statistics.mean(row["direct_worker_cpu_s"] for row in values),
})
(OUT / "profile-summary.json").write_text(
json.dumps({"sampler_runs": sampler_rows, "scope_summary": averaged}, indent=2) + "\n",
encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,46 @@
# Linux78 report
## Passage cost and world identity
The expensive path is `ApplyResolvedStratePassageCarves`: negative density samples enter the passage loop and query the segment index even when their tile has little passage overlap. The route relief calls `VF_FieldPassageWallRelief`, three 3D value-noise octaves at 43/29/19 voxel wavelengths; mouth relief uses the three-octave surface relief. A representative W8 tunnel LOD0 composer census recorded 6.0M composer queries, 18.1M passage iterations, 45.1M segment visits, and 40.8M relief calls.
The retained exact changes are a conservative tile-level upper-mouth candidate list and early return, segment-index min/max rejection before `lower_bound`, and a 64-entry thread-local cache of value-noise lattice corners. The cache preserves the existing interpolation operations and values. The passage share remains high: FI/TN near-LOD passage phase averages 63.0%/63.7% after the change, versus 64.6%/65.9% before; support is 13.8%/13.5% versus 14.1%/13.2%.
The world is exact. The L75 full-grid owner hash and all 24 Clang/GCC × W1/W8/W12 × four-strate density, mesh, payload, and topology comparisons match. No version bump or new captures were needed. Fast-tier passage audits passed.
## Readiness, W8, three interleaved pairs, median seconds
| Strate | LOD0–2 before | LOD0–2 after | Whole view before | Whole view after |
|---|---:|---:|---:|---:|
| SurfaceWorld | 4.36 | 1.76 | 4.36 | 3.43 |
| FloatingIslands | 6.89 | 5.26 | 6.89 | 6.93 |
| TunnelNetwork | 4.88 | 3.22 | 4.88 | 4.55 |
| VerticalShafts | 2.71 | 1.56 | 2.71 | 2.84 |
The headless and Unreal schedulers now submit the collision-critical walkable ring first, then LOD0, LOD1, and LOD2. Far LOD work waits until the requested near LODs are ready and runs at lower priority. The ~1 s near-LOD target is not met for FI/TN; their medians are 5.26 s and 3.22 s. Whole-view times are reported separately above.
## L77 sampler closure and phase census
Values are mean sampled grid-fill/direct grid-fill closure; passage and support are their sampled CPU shares. Near is LOD0–2.
| Strate | Near closure B/A | Whole closure B/A | Passage share near B/A | Support share near B/A |
|---|---:|---:|---:|---:|
| SurfaceWorld | 99.1% / 99.9% | 100.1% / 99.6% | 45.1% / 47.6% | 15.0% / 12.8% |
| FloatingIslands | 99.9% / 99.5% | 99.8% / 99.8% | 64.6% / 63.0% | 14.1% / 13.8% |
| TunnelNetwork | 99.9% / 100.1% | 99.8% / 100.2% | 65.9% / 63.7% | 13.2% / 13.5% |
| VerticalShafts | 100.0% / 100.2% | 99.9% / 100.9% | 45.9% / 42.9% | 5.9% / 6.6% |
Sampler handler cost stayed below 0.05% of profiled thread CPU. Detailed samples and route counts are in `Saved/Linux/l78/sampler-census.csv`, `closure-census.csv`, and `route-census.csv`.
## Builds and checks
- Fast tier: PASS, 366.8 s.
- Hash matrix: PASS, 24/24; the pinned L75 full-grid owner hash also matches.
- Linux VFHostEditor: PASS after timing, `-MaxParallelActions=4`, no address-space limit; UBT reported 70.35 s in its parallel executor.
- No installs, commit, push, or stash.
## Failed runs and rejected experiment
- The first fast-tier launch pointed at an output directory without its expected verifier binaries; corrected, then the final fast tier passed.
- L76's broad per-tile passage-loop experiment caused 612 density mismatches and changed mesh, payload, and topology hashes on 24 tiles; L76 rejected it. It is not part of the accepted L76 reference. In L78, per-brick and direct per-sample feature-bound filters also failed the pinned L75 owner hash, so both were removed. The retained path matches the accepted L76 matrix and pinned L75 owner hash.
- The first direct UBT call could not find `dotnet` in PATH; rerunning through the engine `Build.sh` wrapper succeeded.
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Compare L78 exact tile hashes with the retained L76/L77-equivalent matrix."""
from __future__ import annotations
import json
import re
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l78/hashes"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARIES = {
"clang": ROOT / "Saved/Linux/l78/bin/VoxelForgeCoreHeadless-l78-clang",
"gcc": ROOT / "Saved/Linux/l78/bin/VoxelForgeCoreHeadless-l78-gcc",
}
CAMERAS = {
0: (8.0, 16.0, -372.2626953125),
1: (25.8369140625, 3.19921875, -737.0),
2: (19.166015625, -14.2001953125, -753.3740234375),
3: (-20.0625, -5.068359375, -900.353515625),
}
HASH = re.compile(
r"^\[PipelineTileHash\] lod=(\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
r"density=(0x[0-9a-fA-F]+) mesh=(0x[0-9a-fA-F]+) "
r"payload=(0x[0-9a-fA-F]+) topology=(0x[0-9a-fA-F]+)$",
re.M,
)
def read_hashes(path: Path) -> dict[tuple[int, int, int, int], tuple[str, ...]]:
content = path.read_text(encoding="utf-8", errors="replace")
return {
tuple(int(match.group(i)) for i in range(1, 5)):
tuple(value.lower() for value in match.groups()[4:])
for match in HASH.finditer(content)
}
def run(compiler: str, workers: int, strate: int) -> dict:
OUT.mkdir(parents=True, exist_ok=True)
log = OUT / compiler / f"w{workers}" / f"strate{strate}-after.log"
log.parent.mkdir(parents=True, exist_ok=True)
camera = ",".join(f"{value:.12f}" for value in CAMERAS[strate]) + ",0,0"
command = [
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
"timeout", "1740s", str(BINARIES[compiler]), str(DATA),
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
"--force-generate", f"--workers={workers}", "--clip-radius=3",
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
"--stream-hash-tiles",
]
started = time.monotonic()
with log.open("w", encoding="utf-8") as output:
result = subprocess.run(command, cwd=ROOT, stdout=output,
stderr=subprocess.STDOUT, check=False, timeout=1740)
hashes = read_hashes(log)
if result.returncode != 0 or not hashes:
raise SystemExit(f"hash generation failed: {log}; exit={result.returncode}")
reference = ROOT / "Saved/Linux/l76/hashes" / compiler / f"w{workers}" / f"strate{strate}-before.log"
expected = read_hashes(reference)
missing, extra = expected.keys() - hashes.keys(), hashes.keys() - expected.keys()
mismatches = {name: 0 for name in ("density", "mesh", "payload", "topology")}
examples = []
for key in sorted(expected.keys() & hashes.keys()):
for index, name in enumerate(mismatches):
if expected[key][index] != hashes[key][index]:
mismatches[name] += 1
if len(examples) < 12:
examples.append({"tile": key, "field": name,
"before": expected[key][index],
"after": hashes[key][index]})
row = {
"compiler": compiler,
"workers": workers,
"strate": strate,
"exit_code": result.returncode,
"wall_s": round(time.monotonic() - started, 3),
"expected_tiles": len(expected),
"candidate_tiles": len(hashes),
"missing_keys": len(missing),
"extra_keys": len(extra),
"mismatches": mismatches,
"examples": examples,
"equal": not missing and not extra and not any(mismatches.values()),
"reference": str(reference.relative_to(ROOT)),
"log": str(log.relative_to(ROOT)),
}
print("[L78HashCompare] " + json.dumps(row, sort_keys=True), flush=True)
return row
def main() -> int:
missing = [str(path) for path in (DATA, *BINARIES.values()) if not path.is_file()]
if missing:
raise SystemExit("missing input: " + ", ".join(missing))
rows = []
for compiler in ("clang", "gcc"):
for workers in (1, 8, 12):
for strate in range(4):
row = run(compiler, workers, strate)
rows.append(row)
(OUT / "summary-partial.json").write_text(
json.dumps(rows, indent=2) + "\n", encoding="utf-8")
if not row["equal"]:
raise SystemExit(f"hash mismatch; see {row['log']}")
result = {"comparisons": rows, "equal": all(row["equal"] for row in rows)}
(OUT / "summary.json").write_text(json.dumps(result, indent=2) + "\n",
encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
"""Run interleaved L77/L78 cold readiness pairs for the four owner strates."""
from __future__ import annotations
import json
import re
import statistics
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l78/timing"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARIES = {
"before": ROOT / "Standalone/bin/linux/VoxelForgeCoreHeadless-l77-clang",
"after": ROOT / "Saved/Linux/l78/bin/VoxelForgeCoreHeadless-l78-clang-profile",
}
CAMERAS = {
0: (8.0, 16.0, -372.2626953125),
1: (25.8369140625, 3.19921875, -737.0),
2: (19.166015625, -14.2001953125, -753.3740234375),
3: (-20.0625, -5.068359375, -900.353515625),
}
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
LOD_READY = re.compile(
r"^\[WorldAssemblerL77LODReady\] lod=(\d+) tiles=(\d+) cold_ready_s=([0-9.]+)$",
re.M,
)
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
ORDERS = (("before", "after"), ("after", "before"), ("before", "after"))
def run(strate: int, pair: int, mode: str) -> dict:
OUT.mkdir(parents=True, exist_ok=True)
log = OUT / f"strate{strate}-pair{pair}-{mode}.log"
camera = ",".join(f"{value:.12f}" for value in CAMERAS[strate]) + ",0,0"
command = [
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
"timeout", "1740s", str(BINARIES[mode]), str(DATA),
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
"--workers=8", "--clip-radius=3", "--max-clip-level=8", "--max-level=8",
f"--camera={camera}", "--profile-lod-work", "--linux77-profile",
]
started = time.monotonic()
with log.open("w", encoding="utf-8") as output:
result = subprocess.run(command, cwd=ROOT, stdout=output,
stderr=subprocess.STDOUT, check=False, timeout=1740)
content = log.read_text(encoding="utf-8", errors="replace")
pipeline_match = PIPELINE.search(content)
ready = {int(lod): {"tiles": int(tiles), "seconds": float(seconds)}
for lod, tiles, seconds in LOD_READY.findall(content)}
if result.returncode != 0 or pipeline_match is None or any(lod not in ready for lod in (0, 1, 2)):
raise SystemExit(f"readiness run failed or LOD0-2 data missing: {log}; exit={result.returncode}")
pipeline = dict(FIELDS.findall(pipeline_match.group(1)))
row = {
"strate": strate,
"pair": pair,
"mode": mode,
"exit_code": result.returncode,
"wall_s": round(time.monotonic() - started, 3),
"lod02_ready_s": max(ready[lod]["seconds"] for lod in (0, 1, 2)),
"lod_ready_s": {str(lod): ready[lod]["seconds"] for lod in sorted(ready)},
"whole_view_ready_s": float(pipeline["whole_view_ready"].removesuffix("s")),
"whole_tiles": int(pipeline["whole_tiles"].split("/")[0]),
"sampler": next((line for line in content.splitlines()
if line.startswith("[L77Sampler] action=stop")), None),
"source_stamp": STAMP.search(content).group(1) if STAMP.search(content) else None,
"log": str(log.relative_to(ROOT)),
}
print("[L78TimingRun] " + json.dumps(row, sort_keys=True), flush=True)
return row
def main() -> int:
missing = [str(path) for path in (DATA, *BINARIES.values()) if not path.is_file()]
if missing:
raise SystemExit("missing input: " + ", ".join(missing))
rows = []
for strate in range(4):
for pair, order in enumerate(ORDERS, 1):
paired = {}
for mode in order:
row = run(strate, pair, mode)
rows.append(row)
paired[mode] = row
(OUT / "runs-partial.json").write_text(
json.dumps(rows, indent=2) + "\n", encoding="utf-8")
summary_row = {
"strate": strate,
"pair": pair,
"before_lod02_s": paired["before"]["lod02_ready_s"],
"after_lod02_s": paired["after"]["lod02_ready_s"],
"before_whole_s": paired["before"]["whole_view_ready_s"],
"after_whole_s": paired["after"]["whole_view_ready_s"],
}
print("[L78TimingPair] " + json.dumps(summary_row, sort_keys=True), flush=True)
summary = []
for strate in range(4):
group = [row for row in rows if row["strate"] == strate]
before_near = [row["lod02_ready_s"] for row in group if row["mode"] == "before"]
after_near = [row["lod02_ready_s"] for row in group if row["mode"] == "after"]
before_whole = [row["whole_view_ready_s"] for row in group if row["mode"] == "before"]
after_whole = [row["whole_view_ready_s"] for row in group if row["mode"] == "after"]
summary.append({
"strate": strate,
"pairs": len(before_near),
"before_lod02_s": before_near,
"after_lod02_s": after_near,
"before_lod02_median_s": statistics.median(before_near),
"after_lod02_median_s": statistics.median(after_near),
"before_whole_s": before_whole,
"after_whole_s": after_whole,
"before_whole_median_s": statistics.median(before_whole),
"after_whole_median_s": statistics.median(after_whole),
})
result = {"runs": rows, "summary": summary}
(OUT / "summary.json").write_text(json.dumps(result, indent=2) + "\n",
encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -5061,6 +5061,7 @@ namespace StandaloneStreamBenchmark
*VoxelForgeCore::Linux77Profiler::RouteCount*2>,9> L77DispatchByLOD{}; *VoxelForgeCore::Linux77Profiler::RouteCount*2>,9> L77DispatchByLOD{};
std::array<std::uint64_t,9> L77GridInteriorByLOD{},L77GridHaloByLOD{}; std::array<std::uint64_t,9> L77GridInteriorByLOD{},L77GridHaloByLOD{};
std::array<double,9> L77LODReadySeconds{}; std::array<double,9> L77LODReadySeconds{};
std::array<double,9> LODReadySeconds{};
std::array<std::uint64_t,9> SkirtTrianglesByLOD{}; std::array<std::uint64_t,9> SkirtTrianglesByLOD{};
std::array<std::uint64_t,9> TileCountByLOD{}; std::array<std::uint64_t,9> TileCountByLOD{};
std::array<std::uint64_t,9> ClassifiedAirTilesByLOD{},ClassifiedSolidTilesByLOD{}; std::array<std::uint64_t,9> ClassifiedAirTilesByLOD{},ClassifiedSolidTilesByLOD{};
@@ -5129,6 +5130,9 @@ namespace StandaloneStreamBenchmark
const std::size_t LOD=static_cast<std::size_t>(Tile.Key.Level); const std::size_t LOD=static_cast<std::size_t>(Tile.Key.Level);
const auto& TileDiagnostics=Tile.CoreDiagnostics; const auto& TileDiagnostics=Tile.CoreDiagnostics;
++TileCountByLOD[LOD]; ++TileCountByLOD[LOD];
if(TileCountByLOD[LOD]==ExpectedTilesByLOD[LOD])
LODReadySeconds[LOD]=std::chrono::duration<double>(
Tile.CompletedAt-ColdStart).count();
if(bLinux77Profile) if(bLinux77Profile)
{ {
const auto& D=TileDiagnostics; const auto& D=TileDiagnostics;
@@ -5551,6 +5555,23 @@ namespace StandaloneStreamBenchmark
} }
} }
} }
double NearLODReadySeconds=0.0;
uint64 NearLODExpected=0,NearLODCompleted=0;
for(std::size_t LOD=0;LOD<=2;++LOD)
{
NearLODExpected+=ExpectedTilesByLOD[LOD];
NearLODCompleted+=TileCountByLOD[LOD];
if(ExpectedTilesByLOD[LOD]!=0)
NearLODReadySeconds=std::max(NearLODReadySeconds,LODReadySeconds[LOD]);
}
std::printf("[WorldAssemblerNearLODReady] lods=0-2 tiles=%llu/%llu cold_ready_s=%.6f\n",
static_cast<unsigned long long>(NearLODCompleted),
static_cast<unsigned long long>(NearLODExpected),NearLODReadySeconds);
for(std::size_t LOD=0;LOD<TileCountByLOD.size();++LOD)
if(ExpectedTilesByLOD[LOD]!=0)
std::printf("[WorldAssemblerLODReady] lod=%zu tiles=%llu cold_ready_s=%.6f\n",
LOD,static_cast<unsigned long long>(TileCountByLOD[LOD]),
LODReadySeconds[LOD]);
Streamer.Stop(); Streamer.Stop();
Streamer.DumpColdTimelineTrace(); Streamer.DumpColdTimelineTrace();
bool bScreenSpaceGatePassing=true; bool bScreenSpaceGatePassing=true;
+16 -7
View File
@@ -581,6 +581,7 @@ public:
Workers.clear(); Workers.clear();
std::lock_guard<std::mutex> Lock(Mutex); std::lock_guard<std::mutex> Lock(Mutex);
Ready.clear();Outstanding.clear();Desired.clear();Running.clear(); Ready.clear();Outstanding.clear();Desired.clear();Running.clear();
NearOutstandingJobs=0;
} }
FStandaloneStreamingUpdate UpdateDesired( FStandaloneStreamingUpdate UpdateDesired(
@@ -680,6 +681,7 @@ public:
{ {
if(Desired.contains(Job.Key))return false; if(Desired.contains(Job.Key))return false;
Outstanding.erase(Job.Key);++Stats.CancelledQueuedJobs; Outstanding.erase(Job.Key);++Stats.CancelledQueuedJobs;
if(Job.Key.Level<=2&&NearOutstandingJobs>0)--NearOutstandingJobs;
++CancelledByCrossing[Job.RequestedCrossing];return true; ++CancelledByCrossing[Job.RequestedCrossing];return true;
}); });
Queue.erase(QueueEnd,Queue.end()); Queue.erase(QueueEnd,Queue.end());
@@ -774,15 +776,14 @@ private:
std::vector<FVoxelTileKey> Prioritized=NewDesired.Tiles; std::vector<FVoxelTileKey> Prioritized=NewDesired.Tiles;
if(IsCoarseFirstSchedulingEnabled()) if(IsCoarseFirstSchedulingEnabled())
{ {
// The local walkable ring leads, the coarse enclosure follows, then target-detail work. // Finish the local walkable ring, then all near LODs, before admitting far work.
// L'anneau praticable local passe d'abord, puis l'enveloppe grossiere, puis le detail cible. // Terminer l'anneau praticable, puis tous les LOD proches avant le champ lointain.
const int32 FirstCoarseLevel=FMath::Max(2,World->FullResClipLevels);
auto Tier=[&](const FVoxelTileKey& Key) auto Tier=[&](const FVoxelTileKey& Key)
{ {
if(Key.Level==0&&std::abs(Key.Coord.X-NewDesired.CenterChunk.X)<=1 if(Key.Level==0&&std::abs(Key.Coord.X-NewDesired.CenterChunk.X)<=1
&&std::abs(Key.Coord.Y-NewDesired.CenterChunk.Y)<=1 &&std::abs(Key.Coord.Y-NewDesired.CenterChunk.Y)<=1
&&std::abs(Key.Coord.Z-NewDesired.CenterChunk.Z)<=1)return 0; &&std::abs(Key.Coord.Z-NewDesired.CenterChunk.Z)<=1)return 0;
return Key.Level>=FirstCoarseLevel?1:2; return Key.Level<=2?1:2;
}; };
auto IsAhead=[&](const FVoxelTileKey& Key) auto IsAhead=[&](const FVoxelTileKey& Key)
{ {
@@ -797,7 +798,8 @@ private:
{ {
const int32 TierA=Tier(A),TierB=Tier(B); const int32 TierA=Tier(A),TierB=Tier(B);
if(TierA!=TierB)return TierA<TierB; if(TierA!=TierB)return TierA<TierB;
if(TierA==1&&A.Level!=B.Level)return A.Level>B.Level; if(TierA==1&&A.Level!=B.Level)return A.Level<B.Level;
if(TierA==2&&A.Level!=B.Level)return A.Level>B.Level;
const bool bAheadA=IsAhead(A),bAheadB=IsAhead(B); const bool bAheadA=IsAhead(A),bAheadB=IsAhead(B);
if(bAheadA!=bAheadB)return bAheadA; if(bAheadA!=bAheadB)return bAheadA;
const double DA=StandaloneWorldMesher::TileDistanceSquaredChunks(A,NewDesired.CenterChunk); const double DA=StandaloneWorldMesher::TileDistanceSquaredChunks(A,NewDesired.CenterChunk);
@@ -844,6 +846,7 @@ private:
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Queued,Key,-1, RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Queued,Key,-1,
Queue.size(),Running.size(),Job.PriorityRank,Job.QueuedAt,Job.QueuedAt); Queue.size(),Running.size(),Job.PriorityRank,Job.QueuedAt,Job.QueuedAt);
Outstanding.insert(Key);++Stats.NewJobs; Outstanding.insert(Key);++Stats.NewJobs;
if(Key.Level<=2)++NearOutstandingJobs;
} }
CurrentDesired=std::move(NewDesired); CurrentDesired=std::move(NewDesired);
std::make_heap(Queue.begin(),Queue.end(),FStandaloneTileJobHeapOrder{}); std::make_heap(Queue.begin(),Queue.end(),FStandaloneTileJobHeapOrder{});
@@ -1025,10 +1028,13 @@ private:
size_t DispatchQueueDepth=0,DispatchActiveWorkers=0; size_t DispatchQueueDepth=0,DispatchActiveWorkers=0;
{ {
std::unique_lock<std::mutex> Lock(Mutex); std::unique_lock<std::mutex> Lock(Mutex);
Wake.wait(Lock,[&](){return bStop||!PriorityTasks.empty()||!Queue.empty();}); Wake.wait(Lock,[&](){return bStop||!PriorityTasks.empty()
||(!Queue.empty()&&(NearOutstandingJobs==0||Queue.front().Key.Level<=2));});
if(bStop)return; if(bStop)return;
const bool bNearTileEligible=!Queue.empty()
&&(NearOutstandingJobs==0||Queue.front().Key.Level<=2);
const bool bPreferPriority=!PriorityTasks.empty() const bool bPreferPriority=!PriorityTasks.empty()
&&(Queue.empty()||ConsecutivePriorityTasks<PriorityTaskBurst); &&(!bNearTileEligible||ConsecutivePriorityTasks<PriorityTaskBurst);
if(bPreferPriority) if(bPreferPriority)
{ {
PriorityTask=std::move(PriorityTasks.front());PriorityTasks.pop_front(); PriorityTask=std::move(PriorityTasks.front());PriorityTasks.pop_front();
@@ -1057,6 +1063,7 @@ private:
std::lock_guard<std::mutex> Lock(Mutex); std::lock_guard<std::mutex> Lock(Mutex);
if(bStop)return; if(bStop)return;
Running.erase(Tile.Key); Running.erase(Tile.Key);
if(Tile.Key.Level<=2&&NearOutstandingJobs>0)--NearOutstandingJobs;
if(bColdTimelineTrace) if(bColdTimelineTrace)
RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Finish,Tile.Key, RecordTimelineEvent(FStandaloneCpuTileTimelineEvent::EType::Finish,Tile.Key,
WorkerId,Queue.size(),Running.size(),Job.PriorityRank,Tile.CompletedAt, WorkerId,Queue.size(),Running.size(),Job.PriorityRank,Tile.CompletedAt,
@@ -1065,6 +1072,7 @@ private:
Tile.PayloadBuildMilliseconds); Tile.PayloadBuildMilliseconds);
Ready.push_back(std::move(Tile)); Ready.push_back(std::move(Tile));
} }
Wake.notify_all();
} }
} }
@@ -1208,6 +1216,7 @@ private:
mutable std::mutex Mutex; mutable std::mutex Mutex;
std::condition_variable Wake; std::condition_variable Wake;
bool bStop=true; bool bStop=true;
size_t NearOutstandingJobs=0;
std::vector<FStandaloneWorkerThread> Workers; std::vector<FStandaloneWorkerThread> Workers;
std::deque<std::function<void()>> PriorityTasks; std::deque<std::function<void()>> PriorityTasks;
std::vector<FStandaloneTileJob> Queue; std::vector<FStandaloneTileJob> Queue;