streaming: split dense passage-junction grid fills; heavy-first order + far-parent deferral in Unreal (L85)

Tiles with passage-overlap score >= 9 split grid fill over up to 4 helper lanes (disjoint rows,
same per-sample order, join before MC) when workers are idle. FI junction tile fill 173 -> 99 ms;
estimated serial floor ~0.55-0.60 s. Readiness essentially unchanged (FI/TN LOD0-2 ~2.15 s @W12):
the limit is now aggregate work, not single tiles.
Unreal: LOD0-2 ordered by passage overlap before travel/distance (L80 modes kept); far parents
built after LOD0-1, LOD2 gated on them; helper budget via ParallelFor. Unreal null-RHI FI
arrival: LOD0-2 resident at 3.65 s.
All tile hashes identical to L84 (four whole views). Fast tier PASS; Linux UE host builds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed 2026-10-28 02:02:32 +01:00
1 parent 0a98da6fd0
commit 06f558dc91
17 files changed
+1560 -53

No files matched your search

@@ -15,9 +15,73 @@
#include "HAL/IConsoleManager.h"
#include "Misc/CommandLine.h"
#include <atomic>
#include <array>
namespace
{
struct FVoxelMesherThreadState
{
int32 OctaveBias=0,SampleStep=1,TileCells=0,TileReachBlocks=0,TileReachBlockIndex=INDEX_NONE;
FIntVector TileOrigin=FIntVector::ZeroValue;
uint8 TilePostReachFlags=VoxelGenLOD::AllTileReach;
bool bTilePostReachDebug=false,bTilePostReachBypass=false;
bool bTileReachDiagnosticTile=false,bTileCoreReachProofEnabled=false;
bool bTileCoreReachDecisionRecorded=false,bTileBlockReachValid=false;
float TileReachScale=1.0f;
std::array<uint8,VoxelGenLOD::TileReachMaxBlockCount> TileBlockReachFlags{};
static FVoxelMesherThreadState Capture()
{
FVoxelMesherThreadState State;
State.OctaveBias=VoxelGenLOD::OctaveBias;
State.SampleStep=VoxelGenLOD::SampleStep;
State.TileOrigin=VoxelGenLOD::TileOriginVoxels;
State.TileCells=VoxelGenLOD::TileCellsPerAxis;
State.TilePostReachFlags=VoxelGenLOD::TilePostReachFlags;
State.bTilePostReachDebug=VoxelGenLOD::bTilePostReachDebug;
State.bTilePostReachBypass=VoxelGenLOD::bTilePostReachBypass;
State.bTileReachDiagnosticTile=VoxelGenLOD::bTileReachDiagnosticTile;
State.bTileCoreReachProofEnabled=VoxelGenLOD::bTileCoreReachProofEnabled;
State.bTileCoreReachDecisionRecorded=VoxelGenLOD::bTileCoreReachDecisionRecorded;
State.bTileBlockReachValid=VoxelGenLOD::bTileBlockReachValid;
State.TileReachScale=VoxelGenLOD::TileReachScale;
State.TileReachBlocks=VoxelGenLOD::TileReachBlocksPerAxis;
State.TileReachBlockIndex=VoxelGenLOD::TileReachBlockIndex;
std::copy(std::begin(VoxelGenLOD::TileBlockReachFlags),
std::end(VoxelGenLOD::TileBlockReachFlags),State.TileBlockReachFlags.begin());
return State;
}
void Apply() const
{
VoxelGenLOD::OctaveBias=OctaveBias;
VoxelGenLOD::SampleStep=SampleStep;
VoxelGenLOD::TileOriginVoxels=TileOrigin;
VoxelGenLOD::TileCellsPerAxis=TileCells;
VoxelGenLOD::TilePostReachFlags=TilePostReachFlags;
VoxelGenLOD::bTilePostReachDebug=bTilePostReachDebug;
VoxelGenLOD::bTilePostReachBypass=bTilePostReachBypass;
VoxelGenLOD::bTileReachDiagnosticTile=bTileReachDiagnosticTile;
VoxelGenLOD::bTileCoreReachProofEnabled=bTileCoreReachProofEnabled;
VoxelGenLOD::bTileCoreReachDecisionRecorded=bTileCoreReachDecisionRecorded;
VoxelGenLOD::bTileBlockReachValid=bTileBlockReachValid;
VoxelGenLOD::TileReachScale=TileReachScale;
VoxelGenLOD::TileReachBlocksPerAxis=TileReachBlocks;
VoxelGenLOD::TileReachBlockIndex=TileReachBlockIndex;
std::copy(TileBlockReachFlags.begin(),TileBlockReachFlags.end(),
std::begin(VoxelGenLOD::TileBlockReachFlags));
}
};
struct FVoxelMesherThreadStateScope
{
explicit FVoxelMesherThreadStateScope(const FVoxelMesherThreadState& InTarget)
:Previous(FVoxelMesherThreadState::Capture())
{ InTarget.Apply(); }
~FVoxelMesherThreadStateScope(){Previous.Apply();}
FVoxelMesherThreadState Previous;
};
int32 GVoxelForgeUseOperatorBlock = 1;
FAutoConsoleVariableRef CVarVoxelForgeUseOperatorBlock(
TEXT("voxel.UseOperatorBlock"),
@@ -692,6 +756,7 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
// capacité, donc plus de malloc/free de ~170 Ko (35³ floats) par tuile.
static thread_local TArray<float> DensityGrid;
DensityGrid.SetNumUninitialized(MDim * MDim * MDim);
float* const DensityGridData=DensityGrid.GetData();
const FVoxelSharedDensityGrid* ReuseGrid = InSharedDensityGrid != nullptr
? InSharedDensityGrid : SharedDensityGrid;
const int32 SharedDeltaX = ReuseGrid
@@ -739,7 +804,15 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
const bool bUseOperatorBlock = !bUseSharedDensityGrid
&& !bUseParameterizedBrickEvaluator
&& GVoxelForgeUseOperatorBlock != 0;
if (bUseOperatorBlock)
const int32 GridZSampleCount=GzHi-GzLo+1;
const std::size_t GridFillLaneCount=std::min<std::size_t>({
GridFillExecution.WorkerLimit,16,
static_cast<std::size_t>(FMath::Max(0,GridZSampleCount))});
const bool bUseParallelDensityGrid=GridFillExecution.ParallelFor!=nullptr
&&GridFillLaneCount>1&&GridFillExecution.BeginWorkerContext!=nullptr
&&GridFillExecution.EndWorkerContext!=nullptr&&!bUseSharedDensityGrid
&&!bUseParameterizedBrickEvaluator&&OutCaptureGrid==nullptr;
if (bUseOperatorBlock&&!bUseParallelDensityGrid)
{
// The grid includes one sample of halo on every side. The generator groups these exact
// lattice points by chunk key and lets the prepared op stack fill each group once.
@@ -889,6 +962,80 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
}
#endif
}
else if(bUseParallelDensityGrid)
{
const FVoxelMesherThreadState TileThreadState=FVoxelMesherThreadState::Capture();
std::array<int64,16> LaneSampleCounts{};
std::atomic<bool> bAborted{false};
auto RunLane=[&](std::size_t Lane,std::size_t Worker)
{
void* WorkerState=GridFillExecution.BeginWorkerContext(
GridFillExecution.WorkerContext,Worker);
int64 LaneSamples=0;
{
FVoxelMesherThreadStateScope ThreadStateScope(TileThreadState);
const int32 FirstGz=GzLo+static_cast<int32>(
(static_cast<std::size_t>(GridZSampleCount)*Lane)/GridFillLaneCount);
const int32 EndGz=GzLo+static_cast<int32>(
(static_cast<std::size_t>(GridZSampleCount)*(Lane+1))/GridFillLaneCount);
if(bUseOperatorBlock)
{
Generator->BeginDensityBlock(
OriginVoxels+FIntVector(-Step,-Step,FirstGz*Step),Step,
MDim,MDim,EndGz-FirstGz);
}
for(int32 gz=FirstGz;gz<EndGz;++gz)
{
if(ShouldAbortWork())
{ bAborted.store(true,std::memory_order_relaxed);break; }
for(int32 gy=-1;gy<=GridDim;++gy)
{
if(ShouldAbortWork())
{ bAborted.store(true,std::memory_order_relaxed);break; }
for(int32 gx=-1;gx<=GridDim;++gx)
{
const int32 LocalIndex=((gz+1)*MDim+(gy+1))*MDim+(gx+1);
const int32 ReachBlockX=FMath::Clamp(gx,0,CellsPerAxis-1)
/VoxelGenLOD::TileReachBlockCells;
const int32 ReachBlockY=FMath::Clamp(gy,0,CellsPerAxis-1)
/VoxelGenLOD::TileReachBlockCells;
const int32 ReachBlockZ=FMath::Clamp(gz,0,CellsPerAxis-1)
/VoxelGenLOD::TileReachBlockCells;
VoxelGenLOD::TileReachBlockIndex=
((ReachBlockZ*ReachBlocksPerAxis)+ReachBlockY)
*ReachBlocksPerAxis+ReachBlockX;
float PreEditDensity=0.0f;
DensityGridData[LocalIndex]=Generator->GetDensityAt(
OriginVoxels.X+gx*Step,OriginVoxels.Y+gy*Step,
OriginVoxels.Z+gz*Step,
bCapturePreEditGrid?&PreEditDensity:nullptr);
if(bCapturePreEditGrid)
OutPreEditDensityGrid->Samples[LocalIndex]=PreEditDensity;
++LaneSamples;
}
}
if(bAborted.load(std::memory_order_relaxed))break;
}
if(bUseOperatorBlock)Generator->EndDensityBlock();
}
LaneSampleCounts[Lane]=LaneSamples;
GridFillExecution.EndWorkerContext(
GridFillExecution.WorkerContext,WorkerState,Worker);
};
struct FDispatchContext{decltype(RunLane)* Run=nullptr;};
FDispatchContext DispatchContext{&RunLane};
const auto DispatchLane=[](void* RawContext,std::size_t Lane,std::size_t Worker)
{
auto& Context=*static_cast<FDispatchContext*>(RawContext);
(*Context.Run)(Lane,Worker);
};
GridFillExecution.ParallelFor(GridFillExecution.SchedulerContext,
GridFillLaneCount,GridFillLaneCount,&DispatchContext,DispatchLane);
if(OutDensitySampleCount)
for(std::size_t Lane=0;Lane<GridFillLaneCount;++Lane)
*OutDensitySampleCount+=LaneSampleCounts[Lane];
if(bAborted.load(std::memory_order_relaxed))return FVoxelMeshData();
}
else for (int32 gz = GzLo; gz <= GzHi; gz++)
{
if (ShouldAbortWork())
@@ -959,7 +1106,7 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
GridSampleCount * sizeof(float));
}
}
if (bUseOperatorBlock)
if (bUseOperatorBlock&&!bUseParallelDensityGrid)
{
Generator->EndDensityBlock();
}
+142 -10
View File
@@ -163,6 +163,10 @@ public:
std::atomic<bool> bSnapshotReady{false};
std::atomic<bool> bReady{false};
std::atomic<bool> bInitialFailed{false};
std::atomic<bool> bFarWorldRequired{false};
std::atomic<bool> bFarWorldReady{false};
std::atomic<bool> bFarWorldBuildInFlight{false};
std::atomic<bool> bFarWorldBuildFailed{false};
std::atomic<bool> bRegionCompileInFlight{false};
std::atomic<bool> bRegionCompileFinished{false};
std::atomic<bool> bRegionCompileSucceeded{false};
@@ -7694,6 +7698,10 @@ bool AVoxelWorld::InitializeVoxelForgeCoreWorld(FString& OutFailureReason)
MakeShared<FVoxelForgeCoreRuntimeState, ESPMode::ThreadSafe>();
State->World = std::make_unique<VoxelForgeCore::World>(std::move(*Snapshot));
State->FullResolutionLodCount = State->World->configuration().fullResolutionLodCount;
const bool bFarWorldRequired=State->World->configuration().maxLod
>=State->FullResolutionLodCount;
State->bFarWorldRequired.store(bFarWorldRequired,std::memory_order_release);
State->bFarWorldReady.store(!bFarWorldRequired,std::memory_order_release);
FVector CameraVoxels = FVector::ZeroVector;
FVector InitialPlayerPosition = FVector::ZeroVector;
@@ -7766,7 +7774,7 @@ bool AVoxelWorld::InitializeVoxelForgeCoreWorld(FString& OutFailureReason)
Request.clipMaxVoxel = ClipMax;
Request.workerLimit = static_cast<std::uint32_t>(FMath::Clamp(GetMaxConcurrentTasks(), 1, 16));
Request.detail = VoxelForgeCore::DetailVariant::coherent;
Request.includeFarWorld = true;
Request.includeFarWorld = false;
VoxelForgeCoreState = State;
State->bSnapshotReady.store(true, std::memory_order_release);
@@ -7802,6 +7810,9 @@ bool AVoxelWorld::InitializeVoxelForgeCoreWorld(FString& OutFailureReason)
if (State->World)
{
State->World->retainTileRegions(InitialDesired);
if(State->bFarWorldRequired.load(std::memory_order_acquire))
State->bFarWorldReady.store(State->World->hasFarWorld(),
std::memory_order_release);
FScopeLock RetainedLock(&State->RetainedTilesLock);
for (const VoxelForgeCore::TileKey& Key : InitialDesired)
{
@@ -12365,6 +12376,15 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
const auto SubmitTileWithTraceCount = [&](const FVoxelTileKey& Tile,
EVoxelTileWorkPriority Priority)
{
if(bUseVoxelForgeCore&&Tile.Level>=2)
{
const TSharedPtr<FVoxelForgeCoreRuntimeState,ESPMode::ThreadSafe> CoreState=
VoxelForgeCoreState;
if(CoreState.IsValid()
&&CoreState->bFarWorldRequired.load(std::memory_order_acquire)
&&!CoreState->bFarWorldReady.load(std::memory_order_acquire))
return;
}
if (Tile.Level >= 0 && Tile.Level < 9)
{
++SchedulerLoadAttemptsByLevel[Tile.Level];
@@ -12418,6 +12438,8 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
{
MaxDesiredLevel = FMath::Max(MaxDesiredLevel, T.Level);
}
const TSharedPtr<FVoxelForgeCoreRuntimeState,ESPMode::ThreadSafe> CoreState=
VoxelForgeCoreState;
auto SubmitLevel = [&](int32 Level, EVoxelTileWorkPriority Priority, int32 AdmissionCap,
bool bPreferTravelDirection)
@@ -12437,20 +12459,50 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
}
// Sort only outstanding keys; re-sorting loaded or pending tiles on every tick inflated game-thread work.
// Trier uniquement les cles restantes ; re-trier chaque tick les tuiles chargees ou en attente alourdissait le thread de jeu.
if (bPreferTravelDirection && LevelTiles.Num() > 1)
TMap<FVoxelTileKey,uint32> PassageScores;
if(Level<=2&&CoreState.IsValid()&&CoreState->bReady.load(std::memory_order_acquire))
{
FVector Heading = GetActorTransform().InverseTransformVectorNoScale(PlayerHeading);
if (Heading.IsNearlyZero() && IsValid(PlayerPawn))
FReadScopeLock WorldReadLock(CoreState->WorldLock);
if(CoreState->World)
{
Heading = GetActorTransform().InverseTransformVectorNoScale(
PassageScores.Reserve(LevelTiles.Num());
for(const FVoxelTileKey& Key:LevelTiles)
{
const VoxelForgeCore::TileKey CoreKey{
{Key.Coord.X,Key.Coord.Y,Key.Coord.Z},Key.Level};
PassageScores.Add(Key,CoreState->World->passageOverlapScore(CoreKey));
}
}
}
const bool bSortForPassagePriority=Level<=2&&!PassageScores.IsEmpty();
if ((bPreferTravelDirection||bSortForPassagePriority)&&LevelTiles.Num()>1)
{
FVector Heading=FVector::ZeroVector;
FVector PlayerLocalCm=FVector::ZeroVector;
double TileSizeCm=0.0;
if(bPreferTravelDirection)
{
Heading=GetActorTransform().InverseTransformVectorNoScale(PlayerHeading);
if(Heading.IsNearlyZero()&&IsValid(PlayerPawn))
{
Heading=GetActorTransform().InverseTransformVectorNoScale(
PlayerPawn->GetActorForwardVector());
}
Heading = Heading.GetSafeNormal();
const FVector PlayerLocalCm = WorldToLocalCm(CenterPosition);
const double TileSizeCm = static_cast<double>(VF_ClipLevelExtentVoxels(Level))
* GetActiveVoxelSizeCm();
LevelTiles.Sort([&](const FVoxelTileKey& A, const FVoxelTileKey& B)
Heading=Heading.GetSafeNormal();
PlayerLocalCm=WorldToLocalCm(CenterPosition);
TileSizeCm=static_cast<double>(VF_ClipLevelExtentVoxels(Level))
*GetActiveVoxelSizeCm();
}
std::stable_sort(LevelTiles.GetData(),LevelTiles.GetData()+LevelTiles.Num(),
[&](const FVoxelTileKey& A,const FVoxelTileKey& B)
{
if(bSortForPassagePriority)
{
const uint32 ScoreA=PassageScores.FindRef(A);
const uint32 ScoreB=PassageScores.FindRef(B);
if(ScoreA!=ScoreB)return ScoreA>ScoreB;
}
if(!bPreferTravelDirection)return false;
const FVector CenterA(
(static_cast<double>(A.Coord.X) + 0.5) * TileSizeCm,
(static_cast<double>(A.Coord.Y) + 0.5) * TileSizeCm,
@@ -12509,6 +12561,64 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
}
return true;
};
const auto AreEarlyLODTilesReady = [this]()
{
for(const FVoxelTileKey& Tile:DesiredSorted)
{
if(Tile.Level<=1&&!LoadedTiles.Contains(Tile)
&&!CollisionNotRequiredTiles.Contains(Tile))
return false;
}
return true;
};
if(bUseVoxelForgeCore&&CoreState.IsValid()
&&CoreState->bFarWorldRequired.load(std::memory_order_acquire)
&&CoreState->bReady.load(std::memory_order_acquire)
&&!CoreState->bFarWorldReady.load(std::memory_order_acquire)
&&!CoreState->bFarWorldBuildFailed.load(std::memory_order_acquire)
&&!CoreState->bFarWorldBuildInFlight.load(std::memory_order_acquire)
&&AreEarlyLODTilesReady())
{
bool bExpected=false;
if(CoreState->bFarWorldBuildInFlight.compare_exchange_strong(
bExpected,true,std::memory_order_acq_rel))
{
ActiveTaskCount.fetch_add(1,std::memory_order_relaxed);
UE::Tasks::Launch(TEXT("VoxelForgeCoreFarWorldBuild"),[this,CoreState]()
{
struct FTaskGuard
{
std::atomic<int32>& Counter;
~FTaskGuard(){Counter.fetch_sub(1,std::memory_order_release);}
} Guard{ActiveTaskCount};
if(bShuttingDown.load(std::memory_order_acquire)
||CoreState->bShuttingDown.load(std::memory_order_acquire))
{
CoreState->bFarWorldBuildInFlight.store(false,std::memory_order_release);
return;
}
bool bBuilt=false;
{
FWriteScopeLock WorldWriteLock(CoreState->WorldLock);
if(CoreState->World)bBuilt=CoreState->World->buildFarWorld();
}
if(bBuilt)
{
CoreState->bFarWorldReady.store(true,std::memory_order_release);
CoreState->bRebuildDesired.store(true,std::memory_order_release);
UE_LOG(LogTemp,Display,
TEXT("[VoxelForgeCoreRuntime] deferred_far_world=ready"));
}
else
{
CoreState->bFarWorldBuildFailed.store(true,std::memory_order_release);
UE_LOG(LogTemp,Error,
TEXT("[VoxelForgeCoreRuntime] deferred far-world build failed"));
}
CoreState->bFarWorldBuildInFlight.store(false,std::memory_order_release);
},UE::Tasks::ETaskPriority::BackgroundNormal);
}
}
// Policy 0 reconstructs L77. Policy 3 enters that order only after sustained fast travel.
const bool bPolicy3FastTravel = SchedulerPolicy == 3 && bSpeedAwareFastMode;
@@ -13306,6 +13416,8 @@ void AVoxelWorld::LoadCoreTile(const FVoxelTileKey& Tile, EVoxelTileWorkPriority
if (!State.IsValid()||State->bInitialFailed.load(std::memory_order_acquire)
||State->bRegionCompileFailed.load(std::memory_order_acquire)
||State->bShuttingDown.load(std::memory_order_acquire))return;
if(Tile.Level>=2&&State->bFarWorldRequired.load(std::memory_order_acquire)
&&!State->bFarWorldReady.load(std::memory_order_acquire))return;
const bool bShapeReady=State->bReady.load(std::memory_order_acquire);
if(!bShapeReady)
{
@@ -13502,6 +13614,26 @@ void AVoxelWorld::LoadCoreTile(const FVoxelTileKey& Tile, EVoxelTileWorkPriority
Options.hashDensity = bCaptureHashes;
Options.legacyDirectMappedColumnCache=bLegacyDirectMappedColumnCache;
Options.memoizeMcGradients=bMemoizeCoreMCGradients;
if(Tile.Level==0)
{
const int32 ActiveStreamingTasks=GetActiveGenerationTaskCount();
const int32 WorkerBudget=FMath::Clamp(
GetMaxConcurrentStreamingTasks(),1,12);
const int32 AvailableGridLanes=FMath::Max(0,
WorkerBudget-FMath::Max(0,ActiveStreamingTasks-1));
const int32 GridFillWorkers=FMath::Min(4,AvailableGridLanes);
if(GridFillWorkers>1)
{
FReadScopeLock WorldReadLock(State->WorldLock);
// Only split dense junction overlaps; lower scores are too common to repay
// the helper-lane and per-thread context setup costs.
if(State->World&&State->World->passageOverlapScore(CoreKey)>=9)
{
Options.gridFillScheduler=&State->Scheduler;
Options.gridFillWorkerLimit=static_cast<std::uint32_t>(GridFillWorkers);
}
}
}
std::string CoreError;
std::optional<VoxelForgeCore::TileMesh> CoreMesh;
{
@@ -13,6 +13,7 @@
#include "CoreMinimal.h"
#include <atomic>
#include <cstddef>
#include "VoxelTypes.h" // Pour FVoxelMeshData, CHUNK_SIZE, VOXEL_SIZE, etc.
#include "VoxelGenerator.h"
#include "VoxelMarchingCubesMesher.generated.h"
@@ -38,6 +39,24 @@ struct FVoxelSharedDensityGrid
}
};
// Host-owned scheduling hooks for a predicted-heavy density-grid fill. Each task owns one
// deterministic Z slab and gets the source tile's worker-local sampling context.
struct FVoxelGridFillExecution
{
using FTask = void(*)(void* taskContext, std::size_t index, std::size_t workerIndex);
using FParallelFor = double(*)(void* schedulerContext, std::size_t count,
std::size_t workerLimit, void* taskContext, FTask task);
using FBeginWorkerContext = void*(*)(void* context, std::size_t workerIndex);
using FEndWorkerContext = void(*)(void* context, void* workerState,
std::size_t workerIndex);
void* SchedulerContext = nullptr;
FParallelFor ParallelFor = nullptr;
std::size_t WorkerLimit = 1;
void* WorkerContext = nullptr;
FBeginWorkerContext BeginWorkerContext = nullptr;
FEndWorkerContext EndWorkerContext = nullptr;
};
// Stable plan data copied onto one worker-owned mesher for hybrid LOD0 generation.
// Donnees stables du plan copiees dans un mailleur de worker pour le mode hybride LOD0.
struct FVoxelHybridSharpFeatureBox
@@ -111,6 +130,8 @@ public:
// GenerateMesh calls finish. A malformed/incomplete lattice is ignored and falls back to
// the canonical generator path rather than changing geometry.
void SetSharedDensityGrid(const FVoxelSharedDensityGrid* InGrid) { SharedDensityGrid = InGrid; }
void SetGridFillExecution(const FVoxelGridFillExecution& InExecution)
{ GridFillExecution = InExecution; }
// The caller owns this mesher instance for one tile task; no shared mutable feature state.
// L'appelant possede cette instance pour une tuile; aucune donnee partagee modifiable.
@@ -137,6 +158,7 @@ private:
const std::atomic<bool>* ShutdownFlag = nullptr;
const FVoxelSharedDensityGrid* SharedDensityGrid = nullptr;
FVoxelGridFillExecution GridFillExecution;
TArray<FVoxelHybridSharpFeatureBox> HybridSharpFeatures;
bool bUseHybridSharpMesher = false;
bool bUseHybridSharpFeatureSnap = false;
@@ -214,6 +214,7 @@ namespace VoxelForgeCore
double classificationExclusiveMilliseconds = 0.0;
double marchingCubesExclusiveMilliseconds = 0.0;
double marchingCubesDensityGridMilliseconds = 0.0;
std::uint32_t marchingCubesGridFillWorkers = 1;
double marchingCubesCellClassificationMilliseconds = 0.0;
double marchingCubesGradientNormalsMilliseconds = 0.0;
double marchingCubesEdgeInterpolationMilliseconds = 0.0;
@@ -1462,6 +1463,7 @@ namespace VoxelForgeCore
static void setThreadColumnInterpolationStrideSubvoxels(std::int64_t stride) noexcept;
static std::int64_t threadColumnInterpolationStrideSubvoxels() noexcept;
static void setThreadColumnCacheLegacyDirectMapped(bool enabled) noexcept;
static bool threadColumnCacheLegacyDirectMapped() noexcept;
static const HeightfieldColumnCacheStats& threadColumnCacheStats() noexcept;
FieldInterval interval(std::int64_t minimumXSubvoxels,
std::int64_t minimumYSubvoxels,
@@ -1823,6 +1825,10 @@ namespace VoxelForgeCore
// Diagnostic/performance control; memoizes exact MC lattice gradients at LOD0.
// Controle de mesure/performance; memorise les gradients MC exacts au LOD0.
bool memoizeMcGradients = true;
// Optional host-owned executor for predicted-heavy density-grid fills.
// Hosts leave this null for ordinary tiles and set a small limit for heavy tiles.
const JobScheduler* gridFillScheduler = nullptr;
std::uint32_t gridFillWorkerLimit = 1;
// LOD8-only planar horizon interior grid; zero follows the MC step, 128 restores S9.
// Grille interieure LOD8 seulement; zero suit la maille MC, 128 restaure S9.
std::int32_t lod8HorizonInteriorStepVoxels = 0;
@@ -1878,7 +1884,7 @@ namespace VoxelForgeCore
bool analyticOnly = true,
bool diagnosticIncludeBaseCoveredTiles = false);
void retainTileRegions(std::span<const TileKey> desiredTiles);
void buildFarWorld();
bool buildFarWorld();
bool isCompiled() const noexcept;
bool hasFarWorld() const noexcept;
ShapeBuildStats buildStats() const;
@@ -1909,6 +1915,9 @@ namespace VoxelForgeCore
std::vector<TileKey> desiredTiles(Vec3d cameraPositionVoxels) const;
static std::vector<TileKey> prioritizeTiles(std::span<const TileKey> tiles,
Vec3i centerChunk);
// Count expanded resolved-passage route bounds intersecting the full tile bounds.
// Hosts use this stable estimate to prioritize and split expensive near tiles.
std::uint32_t passageOverlapScore(const TileKey& key) const noexcept;
std::optional<TileMesh> generateTile(const TileKey& key,
const TileGenerationOptions& options,
std::string& error) const;
+22 -1
View File
@@ -271,11 +271,32 @@ public:
}
ActivePassageTileQueries=this;
}
// Duplicate the immutable per-tile masks onto a density-grid helper worker. Mutable
// segment-range caches remain lane-local so helpers never race with the parent query.
explicit FScopedPassageTileQueries(const FScopedPassageTileQueries& source)
:shape_(source.shape_),minimumSubvoxels_(source.minimumSubvoxels_),
maximumSubvoxels_(source.maximumSubvoxels_),featureIndices_(source.featureIndices_),
stampIndices_(source.stampIndices_),cavernFeatureIndices_(source.cavernFeatureIndices_),
passageRoutes_(source.passageRoutes_),passageColumns_(source.passageColumns_),
passageMouthSupports_(source.passageMouthSupports_),
passageMouthSupportSegments_(source.passageMouthSupportSegments_),
passageSegmentRangeCaches_(source.passageSegmentRangeCaches_.size()),
upperMouthSupportPassages_(source.upperMouthSupportPassages_),
routeSegmentCandidateCount_(source.routeSegmentCandidateCount_),
routeFeatureOverlap_(source.routeFeatureOverlap_),columnZOverlap_(source.columnZOverlap_),
mayContainResolvedBoundaryBand_(source.mayContainResolvedBoundaryBand_),
mayContainTransitionPassageFade_(source.mayContainTransitionPassageFade_),
previous_(ActivePassageTileQueries)
{ ActivePassageTileQueries=this; }
~FScopedPassageTileQueries(){ActivePassageTileQueries=previous_;}
FScopedPassageTileQueries(const FScopedPassageTileQueries&)=delete;
FScopedPassageTileQueries& operator=(const FScopedPassageTileQueries&)=delete;
bool matches(const FStandaloneShapeWorld& shape) const{return shape_==&shape;}
static const FScopedPassageTileQueries* CurrentFor(const FStandaloneShapeWorld& shape)
{
const FScopedPassageTileQueries* current=ActivePassageTileQueries;
return current&&current->shape_==&shape?current:nullptr;
}
bool hasFeature(std::size_t passageIndex,bool columnFeature) const
{
const auto& candidates=columnFeature?passageColumns_:passageRoutes_;
+115 -2
View File
@@ -1084,6 +1084,51 @@ namespace VoxelForgeCore
bool bEnabled=false;
};
struct FGridFillWorkerContext
{
FScopedSharedColumnCaches::FHeightfieldCache* HeightfieldCache=nullptr;
FScopedSharedColumnCaches::FIslandCache* IslandCache=nullptr;
const FStandaloneShapeWorld::FScopedPassageTileQueries* PassageQueries=nullptr;
int32 Lod=0;
bool bLegacyDirectMapped=false;
};
struct FGridFillWorkerState
{
bool bPreviousLegacyDirectMapped=false;
std::unique_ptr<FScopedSharedColumnCaches> SharedCaches;
std::unique_ptr<FStandaloneShapeWorld::FScopedPassageTileQueries> PassageQueries;
std::unique_ptr<VoxelForgeCore::Linux77Profiler::FScopedLod> Lod;
~FGridFillWorkerState()
{
Lod.reset();
PassageQueries.reset();
SharedCaches.reset();
Heightfield2D::setThreadColumnCacheLegacyDirectMapped(
bPreviousLegacyDirectMapped);
}
};
void* BeginGridFillWorkerContext(void* rawContext,std::size_t)
{
const auto& context=*static_cast<FGridFillWorkerContext*>(rawContext);
auto state=std::make_unique<FGridFillWorkerState>();
state->bPreviousLegacyDirectMapped=
Heightfield2D::threadColumnCacheLegacyDirectMapped();
Heightfield2D::setThreadColumnCacheLegacyDirectMapped(context.bLegacyDirectMapped);
state->SharedCaches=std::make_unique<FScopedSharedColumnCaches>(
context.HeightfieldCache,context.IslandCache);
if(context.PassageQueries)
state->PassageQueries=
std::make_unique<FStandaloneShapeWorld::FScopedPassageTileQueries>(
*context.PassageQueries);
state->Lod=std::make_unique<VoxelForgeCore::Linux77Profiler::FScopedLod>(context.Lod);
return state.release();
}
void EndGridFillWorkerContext(void*,void* rawState,std::size_t)
{ delete static_cast<FGridFillWorkerState*>(rawState); }
World::World(ResolvedWorldSnapshot snapshot):impl_(std::make_unique<Impl>())
{
if(snapshot.data_)
@@ -1737,8 +1782,13 @@ namespace VoxelForgeCore
impl_->tileRegionCount.store(impl_->tileRegions.size(),std::memory_order_release);
}
void World::buildFarWorld()
{ if(impl_&&impl_->shapeReady.load(std::memory_order_acquire)&&impl_->shape)impl_->shape->BuildFarWorld(); }
bool World::buildFarWorld()
{
if(!impl_||!impl_->shapeReady.load(std::memory_order_acquire)||!impl_->shape)
return false;
if(!impl_->shape->HasFarWorld())impl_->shape->BuildFarWorld();
return impl_->shape->HasFarWorld();
}
bool World::isCompiled() const noexcept
{ return impl_&&impl_->shapeReady.load(std::memory_order_acquire); }
bool World::hasFarWorld() const noexcept
@@ -1914,6 +1964,41 @@ namespace VoxelForgeCore
return output;
}
std::uint32_t World::passageOverlapScore(const TileKey& key) const noexcept
{
if(!impl_||!impl_->resolvedLayout||key.lod<0||key.lod>30)return 0;
constexpr std::int64_t SubvoxelsPerVoxel=WorldAssemblySubvoxelsPerVoxel;
constexpr std::int64_t ExtraInfluenceVoxels=32;
const std::int64_t Extent=static_cast<std::int64_t>(CoreTileVoxelCount)<<key.lod;
const std::int64_t TileMinimum[3]={
static_cast<std::int64_t>(key.coordinate.x)*Extent,
static_cast<std::int64_t>(key.coordinate.y)*Extent,
static_cast<std::int64_t>(key.coordinate.z)*Extent};
const std::int64_t TileMaximum[3]={TileMinimum[0]+Extent,
TileMinimum[1]+Extent,TileMinimum[2]+Extent};
std::uint32_t Score=0;
for(const auto& Passage:impl_->resolvedLayout->passages)
for(std::size_t Index=1;Index<Passage.controlPoints.size();++Index)
{
const auto& A=Passage.controlPoints[Index-1];
const auto& B=Passage.controlPoints[Index];
const std::int64_t Radius=std::max(A.radiusSubvoxels,B.radiusSubvoxels)
/SubvoxelsPerVoxel+ExtraInfluenceVoxels;
bool bOverlap=true;
for(std::size_t Axis=0;Axis<3;++Axis)
{
const std::int64_t AVoxel=A.positionSubvoxels[Axis]/SubvoxelsPerVoxel;
const std::int64_t BVoxel=B.positionSubvoxels[Axis]/SubvoxelsPerVoxel;
const std::int64_t Minimum=std::min(AVoxel,BVoxel)-Radius;
const std::int64_t Maximum=std::max(AVoxel,BVoxel)+Radius;
bOverlap=bOverlap&&Maximum>=TileMinimum[Axis]
&&Minimum<=TileMaximum[Axis];
}
if(bOverlap)++Score;
}
return Score;
}
std::vector<TileKey> World::desiredTiles(Vec3d cameraPosition) const
{
if(!impl_||!impl_->owner)return {};
@@ -2724,6 +2809,33 @@ namespace VoxelForgeCore
passageFeatureScope.mouthSupportSegments(passageIndex).size();
}
}
FGridFillWorkerContext GridFillWorkerContext;
if(options.gridFillScheduler&&options.gridFillWorkerLimit>1)
{
GridFillWorkerContext.HeightfieldCache=options.useSharedColumnCache
?impl_->sharedHeightfieldColumns.get():nullptr;
GridFillWorkerContext.IslandCache=options.useSharedColumnCache
?impl_->sharedIslandColumns.get():nullptr;
GridFillWorkerContext.PassageQueries=
FStandaloneShapeWorld::FScopedPassageTileQueries::CurrentFor(tileShape);
GridFillWorkerContext.Lod=key.lod;
GridFillWorkerContext.bLegacyDirectMapped=
options.legacyDirectMappedColumnCache||key.lod<=2;
FVoxelGridFillExecution Execution;
Execution.SchedulerContext=const_cast<JobScheduler*>(options.gridFillScheduler);
Execution.ParallelFor=[](void* RawScheduler,std::size_t Count,
std::size_t WorkerLimit,void* TaskContext,FVoxelGridFillExecution::FTask Task)
{
return static_cast<const JobScheduler*>(RawScheduler)->parallel_for(
Count,1,WorkerLimit,TaskContext,Task,nullptr,0);
};
Execution.WorkerLimit=std::clamp<std::size_t>(
options.gridFillWorkerLimit,2,16);
Execution.WorkerContext=&GridFillWorkerContext;
Execution.BeginWorkerContext=&BeginGridFillWorkerContext;
Execution.EndWorkerContext=&EndGridFillWorkerContext;
mesher.SetGridFillExecution(Execution);
}
const bool bDiagnosticColumnSampling=options.diagnosticGlobalOctaveDrop!=0
||options.diagnosticColumnInterpolationStride>1;
const std::uint32_t PreviousOctaveDrop=bDiagnosticColumnSampling
@@ -2781,6 +2893,7 @@ namespace VoxelForgeCore
out.diagnostics.marchingCubesExclusiveMilliseconds
-out.diagnostics.marchingCubesSkirtMilliseconds);
out.diagnostics.marchingCubesDensityGridMilliseconds=McStageTimings.DensityGridMilliseconds;
out.diagnostics.marchingCubesGridFillWorkers=McStageTimings.GridFillWorkerCount;
out.diagnostics.marchingCubesCellClassificationMilliseconds=
McStageTimings.CellClassificationMilliseconds;
out.diagnostics.marchingCubesGradientNormalsMilliseconds=
@@ -2653,6 +2653,9 @@ void Heightfield2D::setThreadColumnCaptureBounds(std::int64_t minimumXSubvoxels,
void Heightfield2D::setThreadColumnCacheLegacyDirectMapped(bool enabled) noexcept
{ world_assembly_detail::HeightfieldColumnCacheLegacyDirectMapped=enabled; }
bool Heightfield2D::threadColumnCacheLegacyDirectMapped() noexcept
{ return world_assembly_detail::HeightfieldColumnCacheLegacyDirectMapped; }
void Heightfield2D::setThreadColumnOctaveDrop(std::uint32_t octaveDrop) noexcept
{ world_assembly_detail::HeightfieldColumnOctaveDrop=octaveDrop; }
@@ -0,0 +1,61 @@
# Linux 85 — split heavy grid fills
## Split and floor
Full-resolution passage tiles with overlap score **9 or higher** can split density-grid fill over up to four helper lanes. Lanes copy the tile’s worker-local sampling caches and passage mask, write disjoint grid rows, and join before marching cubes. Each sample keeps the same evaluation order. The helper budget is limited by active tile workers under the 12-CPU affinity, so saturated W12/W16 work stays serial inside each tile.
Matched W8 profiles show the junction tile `(0,1,-93,lod0)` improved. Times are medians of three L84/L85 profiles; each tile materialized 42,875 samples.
| View | Junction service, L84 → L85 | Junction grid fill, L84 → L85 | Slowest full-grid service / fill, L84 → L85 | Estimated floor, L84 → L85 |
|---|---:|---:|---:|---:|
| FI | 243 → 167 ms | 173 → 99 ms | 243 / 173 → 224 / 167 ms | 0.560 → 0.547 s |
| TN | 168 → 159 ms | 121 → 100 ms | 243 / 172 → 246 / 186 ms | 0.593 → 0.595 s |
Floor = median base-shape time + median far-parent time + median per-run slowest full-grid LOD0 service. The junction fill fell 43% in FI and 18% in TN, but the estimated floor moved only slightly: the slowest tile can be a different, unsplit tile. Readiness also stayed close to L84 overall, with mixed W8 results and nearly unchanged FI/TN W12 results.
Cold LOD0–2-ready / whole-view seconds. L85 uses one warmup then three interleaved W8/W12 pairs per stratum; L84 values are from the L84 report.
| Stratum | L84 W8 | L85 W8 | L84 W12 | L85 W12 |
|---|---:|---:|---:|---:|
| SurfaceWorld | 1.021 / 2.209 | 0.976 / 2.166 | 0.990 / 2.081 | 0.961 / 2.019 |
| FloatingIslands | 2.264 / 3.429 | 2.286 / 3.469 | 2.180 / 3.341 | 2.162 / 3.306 |
| TunnelNetwork | 2.269 / 3.377 | 2.323 / 3.377 | 2.146 / 3.153 | 2.147 / 3.140 |
| VerticalShafts | 1.127 / 2.080 | 1.243 / 2.229 | 1.086 / 2.023 | 1.080 / 2.000 |
## Unreal and brick look
`VoxelWorld.cpp` now orders outstanding LOD0–2 tiles by passage overlap before its existing travel/distance order, while keeping L80 speed-aware modes and the collision-critical ring. Initial compile omits far parents; the Unreal scheduler builds them after desired LOD0–1 tiles are loaded and gates LOD2 until publication. The Core task adapter passes the available helper budget to Unreal `ParallelFor`.
The null-RHI FI-start trace reached 1,216/1,216 resident LOD0–2 tiles at **3.650 s**, before movement began; the center was `(3,0,-93)`. Trace: `Saved/Linux/l85/unreal-fi/fi-arrival-l85.trace.jsonl`.
Per-tile passage masks prune passage candidates at tile scope. They do not prove the sign of the complete density field inside a brick, so they do not make a cheap, sound brick skip available; no new brick audit was run.
## Hashes and checks
L84 and L85 whole-view and per-tile density, mesh, payload and topology hashes match exactly; zero tile mismatches.
| Stratum | Tiles | Density | Mesh | Payload | Topology |
|---|---:|---|---|---|---|
| SurfaceWorld | 2,850 | `40b76d0072a2adcd` | `8559df0850d4d7f1` | `7700965ff31ac702` | `0e7f2a16b4cc71a9` |
| FloatingIslands | 2,249 | `99ca213b21ce86e4` | `2977525405b79fff` | `3d7d2efb581710bd` | `af1916c949224871` |
| TunnelNetwork | 2,233 | `8d23c26f5c092be1` | `15c3aad2e29d9ea0` | `3d48f68d511c21df` | `5b1584676884edd8` |
| VerticalShafts | 2,329 | `876b44981d106be3` | `aaef57cb08bd6845` | `b19617df9f4acbb1` | `e8eee991260adff4` |
Fast tier: **PASS**, 274 checks, 286.5 s. Linux `VFHostEditor`: **PASS**, final post-timing UBT invocation used `-NoUBA -MaxParallelActions=4`, taskset CPUs 0–11, and no address-space limit. The prior full compile built 35/35 actions; three existing RealtimeMesh deprecation warnings remain. The per-tile W8 profile and timing runners used taskset and the 6 GiB address-space limit. Headless processes in hash and fast runs used both limits.
## Source files
- `Source/VoxelForge/Private/VoxelWorld.cpp` — Unreal tile ordering, far-parent deferral and helper-lane budget.
- `Source/VoxelForge/Private/VoxelMarchingCubesMesher.cpp`, `Source/VoxelForge/Public/VoxelMarchingCubesMesher.h` — parallel grid-fill hooks and worker joins.
- `Standalone/Core/include/VoxelForgeCore/VoxelForgeCore.h`, `Standalone/Core/src/VoxelForgeCore.cpp` — scheduler API, worker-local context and overlap scoring.
- `Standalone/Core/src/ShapeWorldInternal.h`, `Standalone/Core/src/WorldAssemblerInternal.h` — cloneable passage mask and cache-mode access.
- `Standalone/Source/StreamingWorld.h`, `Standalone/Source/StreamBenchmarkPipeline.h`, `Standalone/Source/VoxelForge/Private/VoxelMarchingCubesMesher.cpp`, `Standalone/Source/VoxelForge/Public/VoxelMarchingCubesMesher.h` — headless scheduling and split fill.
## Failed runs
- The first split smoke hit SIGBUS because helper threads wrote to their empty thread-local density buffer. Workers now write into the parent-owned grid; final hashes match.
- Splitting every passage overlap was too broad and regressed FI/TN throughput. The final score-9 threshold and active-worker budget narrow it to dense junction tiles with spare capacity.
- The first UBT attempt stopped in UnrealHeaderTool because `<cstddef>` followed the generated header include. Include order was fixed; final builds passed.
- The first fast-tier launch lacked its round-specific binary. Building the final-stamp binary fixed it; the 274-check rerun passed.
No run exceeded 30 minutes. No install, commit, push or stash was made.
@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Compare L84 and L85 per-tile and whole-view hashes for all four full views."""
from __future__ import annotations
import json
import re
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l85/hash-check-final"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARIES = {
"L84": ROOT / "Saved/Linux/l84/bin/VoxelForgeCoreHeadless-l84-clang19",
"L85": ROOT / "Saved/Linux/l85/bin/VoxelForgeCoreHeadless-clang19",
}
STAMPS = {
"L84": "c433a039b0255ce3c4794bf20313b8a5a11442feeb07b6555f585688949741a5",
"L85": "2967bb8aaea5a7beac0cd0735c0c2e5cccd7b31a9622dd33395e81ff1e5f2589",
}
CAMERAS = {
0: (8.0, 16.0, -372.2626953125),
1: (25.8369140625, 3.19921875, -737.0),
2: (19.166015625, -14.2001953125, -753.3740234375),
3: (-20.0625, -5.068359375, -900.353515625),
}
EXPECTED_WHOLE = {0: 2850, 1: 2249, 2: 2233, 3: 2329}
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
TILE_HASH = re.compile(
r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) "
r"topology=(0x[0-9a-f]+)$", re.M)
WHOLE_HASH = re.compile(
r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) "
r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M)
def run(strate: int, version: str) -> dict:
OUT.mkdir(parents=True, exist_ok=True)
log = OUT / f"{version.lower()}-s{strate}.log"
camera = ",".join(f"{v:.12f}" for v in CAMERAS[strate]) + ",0,0"
command = [
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
"env", "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", "timeout", "1740s",
str(BINARIES[version]), str(DATA), "--world-assembler-prototype",
"--stream-benchmark", "--stream-pipeline", "--workers=8", "--clip-radius=3",
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
"--stream-scheduler-policy=3", "--stream-hash-tiles",
]
started = time.monotonic()
with log.open("w", encoding="utf-8") as stream:
result = subprocess.run(command, cwd=ROOT, stdout=stream,
stderr=subprocess.STDOUT, check=False, timeout=1740)
output = log.read_text(encoding="utf-8", errors="replace")
source, whole = STAMP.search(output), WHOLE_HASH.search(output)
hashes = {
(int(match.group(1)), int(match.group(2)), int(match.group(3)), int(match.group(4))):
match.group(5, 6, 7, 8)
for match in TILE_HASH.finditer(output)
}
if result.returncode or source is None or whole is None:
raise RuntimeError(f"hash run failed: exit={result.returncode} {log}")
if source.group(1) != STAMPS[version]:
raise RuntimeError(f"source stamp mismatch in {log}: {source.group(1)}")
if int(whole.group(1)) != EXPECTED_WHOLE[strate] or len(hashes) != EXPECTED_WHOLE[strate]:
raise RuntimeError(f"incomplete whole-view hashes in {log}: tiles={len(hashes)} whole={whole.group(1)}")
row = {
"version": version, "strate": strate, "exit_code": result.returncode,
"wall_s": round(time.monotonic() - started, 3), "tile_count": len(hashes),
"whole_hash": whole.group(1, 2, 3, 4, 5), "hashes": hashes,
"source_stamp": source.group(1), "log": str(log.relative_to(ROOT)),
}
print("[L85HashRun] " + json.dumps(
{key: value for key, value in row.items() if key != "hashes"}, sort_keys=True),
flush=True)
return row
def main() -> int:
for binary in (*BINARIES.values(), DATA):
if not binary.is_file():
raise SystemExit(f"missing input: {binary}")
rows, comparisons = [], []
for strate in range(4):
pair = {version: run(strate, version) for version in ("L84", "L85")}
before, after = pair["L84"], pair["L85"]
mismatches = sorted(key for key in set(before["hashes"]) | set(after["hashes"])
if before["hashes"].get(key) != after["hashes"].get(key))
comparison = {
"strate": strate, "tile_count": len(before["hashes"]),
"tile_hashes_identical": not mismatches,
"whole_hash_identical": before["whole_hash"] == after["whole_hash"],
"mismatch_count": len(mismatches), "first_mismatches": mismatches[:12],
"l84_whole_hash": before["whole_hash"], "l85_whole_hash": after["whole_hash"],
"logs": {version: pair[version]["log"] for version in pair},
}
comparisons.append(comparison)
rows.extend({key: value for key, value in row.items() if key != "hashes"}
for row in pair.values())
(OUT / "runs-partial.json").write_text(json.dumps(
{"comparisons": comparisons, "runs": rows}, indent=2) + "\n", encoding="utf-8")
print("[L85HashComparison] " + json.dumps(comparison, sort_keys=True), flush=True)
return 0 if all(row["tile_hashes_identical"] and row["whole_hash_identical"]
for row in comparisons) else 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""L80-protocol W8/W12 readiness pairs for the L85 split-grid candidate."""
from __future__ import annotations
import json
import re
import statistics
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l85/scaling-final"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARY = ROOT / "Saved/Linux/l85/bin/VoxelForgeCoreHeadless-clang19"
WORKERS = (8, 12)
CAMERAS = {
0: (8.0, 16.0, -372.2626953125),
1: (25.8369140625, 3.19921875, -737.0),
2: (19.166015625, -14.2001953125, -753.3740234375),
3: (-20.0625, -5.068359375, -900.353515625),
}
EXPECTED_NEAR = {0: 1408, 1: 1216, 2: 1344, 3: 1344}
EXPECTED_WHOLE = {0: 2850, 1: 2249, 2: 2233, 3: 2329}
SOURCE_STAMP = "2967bb8aaea5a7beac0cd0735c0c2e5cccd7b31a9622dd33395e81ff1e5f2589"
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
def run(strate: int, workers: int, phase: str, rep: int) -> dict:
OUT.mkdir(parents=True, exist_ok=True)
log = OUT / f"s{strate}-w{workers}-{phase}{rep}.log"
camera = ",".join(f"{v:.12f}" for v in CAMERAS[strate]) + ",0,0"
command = [
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
"env", "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", "timeout", "1740s",
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
"--stream-pipeline", f"--workers={workers}", "--clip-radius=3",
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
"--profile-lod-work", "--stream-scheduler-policy=3",
]
start = time.monotonic()
with log.open("w", encoding="utf-8") as stream:
result = subprocess.run(command, cwd=ROOT, stdout=stream,
stderr=subprocess.STDOUT, check=False, timeout=1740)
output = log.read_text(encoding="utf-8", errors="replace")
pipeline_match, near_match, stamp_match = (
PIPELINE.search(output), NEAR.search(output), STAMP.search(output))
if result.returncode or pipeline_match is None or near_match is None:
raise RuntimeError(f"readiness run failed: exit={result.returncode} {log}")
if stamp_match is None or stamp_match.group(1) != SOURCE_STAMP:
raise RuntimeError(f"source stamp mismatch in {log}")
pipeline = dict(FIELDS.findall(pipeline_match.group(1)))
near_done, near_total = int(near_match.group(1)), int(near_match.group(2))
whole_done, whole_total = (int(v) for v in pipeline["whole_tiles"].split("/"))
if (near_done, near_total) != (EXPECTED_NEAR[strate], EXPECTED_NEAR[strate]):
raise RuntimeError(f"near readiness incomplete in {log}: {near_done}/{near_total}")
if (whole_done, whole_total) != (EXPECTED_WHOLE[strate], EXPECTED_WHOLE[strate]):
raise RuntimeError(f"whole readiness incomplete in {log}: {whole_done}/{whole_total}")
row = {
"strate": strate, "workers": workers, "phase": phase, "replicate": rep,
"exit_code": result.returncode, "wall_s": round(time.monotonic() - start, 3),
"lod02_ready_s": float(near_match.group(3)),
"whole_view_ready_s": float(pipeline["whole_view_ready"].removesuffix("s")),
"shape_s": float(pipeline["shape"].removesuffix("s")),
"far_shape_s": float(pipeline["far_shape"].removesuffix("s")),
"first_tile_ready_s": float(pipeline["first_tile_ready"].removesuffix("s")),
"neighborhood_ready_s": float(pipeline["neighborhood_ready"].removesuffix("s")),
"full_detail_ready_s": float(pipeline["full_detail_ready"].removesuffix("s")),
"near_tiles": near_total, "whole_tiles": whole_total,
"source_stamp": stamp_match.group(1), "log": str(log.relative_to(ROOT)),
}
print("[L85ScalingRun] " + json.dumps(row, sort_keys=True), flush=True)
return row
def main() -> int:
if not BINARY.is_file() or not DATA.is_file():
raise SystemExit(f"missing input: {BINARY} or {DATA}")
rows = []
for strate in range(4):
for workers in WORKERS:
run(strate, workers, "warmup", 0)
for rep, order in enumerate(((8, 12), (12, 8), (8, 12)), 1):
for workers in order:
rows.append(run(strate, workers, "measured", rep))
(OUT / "runs-partial.json").write_text(
json.dumps(rows, indent=2) + "\n", encoding="utf-8")
summary = []
for strate in range(4):
for workers in WORKERS:
subset = [row for row in rows
if row["strate"] == strate and row["workers"] == workers]
item = {"strate": strate, "workers": workers, "measured_runs": len(subset)}
for metric in ("lod02_ready_s", "whole_view_ready_s", "shape_s", "far_shape_s",
"first_tile_ready_s", "neighborhood_ready_s", "full_detail_ready_s"):
values = [row[metric] for row in subset]
item[metric] = values
item[f"{metric}_median"] = statistics.median(values)
item[f"{metric}_min"] = min(values)
item[f"{metric}_max"] = max(values)
summary.append(item)
(OUT / "summary.json").write_text(
json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n",
encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""Interleaved W16 tile and shape-stage profiles for L85."""
from __future__ import annotations
import json
import re
import statistics
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l85/serial-floor-final"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARIES = {
"L85": ROOT / "Saved/Linux/l85/bin/VoxelForgeCoreHeadless-clang19",
}
STAMPS = {
"L85": "2967bb8aaea5a7beac0cd0735c0c2e5cccd7b31a9622dd33395e81ff1e5f2589",
}
CAMERAS = {
1: (25.8369140625, 3.19921875, -737.0),
2: (19.166015625, -14.2001953125, -753.3740234375),
}
NAMES = {1: "FI", 2: "TN"}
EXPECTED_NEAR = {1: 1216, 2: 1344}
EXPECTED_WHOLE = {1: 2249, 2: 2233}
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
SHAPE_FIELDS = re.compile(r"([a-zA-Z0-9_+\-]+)=([0-9.]+)")
TIMELINE = re.compile(
r"^\[CpuTileTimeline\] event=finish key=\(([-0-9]+),([-0-9]+),([-0-9]+),lod([0-9]+)\) (.*)$", re.M)
STAGES = re.compile(
r"^\[CpuTileStages\] key=\(([-0-9]+),([-0-9]+),([-0-9]+),lod([0-9]+)\) (.*)$", re.M)
SHAPE_STAGE = re.compile(r"^\[ShapeStage\] (.*)$", re.M)
FAR_STAGE = re.compile(r"^\[FarShapeStage\] (.*)$", re.M)
def run(strate: int, version: str, rep: int) -> dict:
out_dir = OUT / version.lower()
out_dir.mkdir(parents=True, exist_ok=True)
log = out_dir / f"{NAMES[strate].lower()}-w16-r{rep}.log"
camera = ",".join(f"{v:.12f}" for v in CAMERAS[strate]) + ",0,0"
command = [
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
"env", "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", "timeout", "1740s",
str(BINARIES[version]), str(DATA), "--world-assembler-prototype",
"--stream-benchmark", "--stream-pipeline", "--workers=16", "--clip-radius=3",
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
"--profile-lod-work", "--instrument-tile-drain", "--stream-scheduler-policy=3",
]
started = time.monotonic()
with log.open("w", encoding="utf-8") as stream:
result = subprocess.run(command, cwd=ROOT, stdout=stream,
stderr=subprocess.STDOUT, check=False, timeout=1740)
output = log.read_text(encoding="utf-8", errors="replace")
source = STAMP.search(output)
near = NEAR.search(output)
pipeline = PIPELINE.search(output)
if result.returncode or source is None or near is None or pipeline is None:
raise RuntimeError(f"profile run failed: exit={result.returncode} {log}")
if source.group(1) != STAMPS[version]:
raise RuntimeError(f"source stamp mismatch in {log}: {source.group(1)}")
near_done, near_total = int(near.group(1)), int(near.group(2))
p = dict(FIELDS.findall(pipeline.group(1)))
whole_done, whole_total = (int(value) for value in p["whole_tiles"].split("/"))
if near_total != EXPECTED_NEAR[strate] or near_done != near_total:
raise RuntimeError(f"incomplete near tiles in {log}: {near_done}/{near_total}")
if whole_total != EXPECTED_WHOLE[strate] or whole_done != whole_total:
raise RuntimeError(f"incomplete whole view in {log}: {whole_done}/{whole_total}")
stage_rows = {}
for match in STAGES.finditer(output):
key = tuple(int(value) for value in match.group(1, 2, 3, 4))
stage_rows[key] = dict(FIELDS.findall(match.group(5)))
tiles = []
for match in TIMELINE.finditer(output):
key = tuple(int(value) for value in match.group(1, 2, 3, 4))
fields = dict(FIELDS.findall(match.group(5)))
if key[3] > 2:
continue
stage = stage_rows.get(key, {})
tiles.append({
"key": key, "lod": key[3], "service_ms": float(fields["service_ms"]),
"queued_ms": float(fields["queued_ms"]), "dispatch_ms": float(fields["dispatch_ms"]),
"finish_ms": float(fields["finish_ms"]), "priority_rank": int(fields["priority_rank"]),
"queue_depth": int(fields["queue_depth"]), "active_workers": int(fields["active_workers"]),
"samples": int(stage.get("samples", "0")),
"candidate_cells": int(stage.get("candidate_cells", "0")),
"surface_cells": int(stage.get("surface_cells", "0")),
"tile_class": stage.get("tile_class", "unknown"),
"mc_grid_ms": float(stage.get("mc_grid_ms", "0")),
"mc_grid_workers": int(stage.get("mc_grid_workers", "1")),
"mc_grid_workers": int(stage.get("mc_grid_workers", "1")),
"mc_cell_ms": float(stage.get("mc_cell_ms", "0")),
"mc_gradient_ms": float(stage.get("mc_gradient_ms", "0")),
"mc_skirt_ms": float(stage.get("mc_skirt_ms", "0")),
})
if len(tiles) != near_total:
raise RuntimeError(f"incomplete near tile timeline in {log}: {len(tiles)}/{near_total}")
by_lod = {}
for lod in range(3):
subset = [tile for tile in tiles if tile["lod"] == lod]
services = [tile["service_ms"] for tile in subset]
by_lod[lod] = {
"tiles": len(subset), "p50_service_ms": statistics.median(services),
"p90_service_ms": sorted(services)[int((len(services) - 1) * 0.90)],
"p95_service_ms": sorted(services)[int((len(services) - 1) * 0.95)],
"max_service_ms": max(services),
}
slow = sorted(tiles, key=lambda tile: tile["service_ms"], reverse=True)[:12]
stages = [dict(SHAPE_FIELDS.findall(match.group(1))) for match in SHAPE_STAGE.finditer(output)]
far_stages = [dict(FIELDS.findall(match.group(1))) for match in FAR_STAGE.finditer(output)]
far_shape_seconds = (float(p["far_shape"].removesuffix("s"))
if "far_shape" in p else
(float(far_stages[-1]["total"]) / 1000.0 if far_stages else 0.0))
row = {
"version": version, "strate": strate, "name": NAMES[strate], "replicate": rep,
"workers": 16, "exit_code": result.returncode,
"wall_s": round(time.monotonic() - started, 3),
"near_ready_s": float(near.group(3)), "pipeline_shape_s": float(p["shape"].removesuffix("s")),
"pipeline_far_shape_s": far_shape_seconds,
"shape_stages": stages, "far_shape_stages": far_stages,
"tile_count": len(tiles), "by_lod": by_lod, "slow_tiles": slow,
"source_stamp": source.group(1), "log": str(log.relative_to(ROOT)),
}
print("[L85SerialFloorRun] " + json.dumps(row, sort_keys=True), flush=True)
return row
def main() -> int:
for binary in BINARIES.values():
if not binary.is_file():
raise SystemExit(f"missing binary: {binary}")
rows = []
for rep in range(1, 4):
order = (1, 2) if rep % 2 else (2, 1)
for strate in order:
rows.append(run(strate, "L85", rep))
(OUT / "runs-partial.json").write_text(
json.dumps(rows, indent=2) + "\n", encoding="utf-8")
summary = []
for strate in (1, 2):
for version in ("L85",):
subset = [row for row in rows
if row["strate"] == strate and row["version"] == version]
item = {"strate": strate, "name": NAMES[strate], "version": version,
"runs": [row["log"] for row in subset]}
for metric in ("near_ready_s", "pipeline_shape_s", "pipeline_far_shape_s"):
values = [row[metric] for row in subset]
item[f"{metric}_runs"] = values
item[f"{metric}_median"] = statistics.median(values)
item[f"{metric}_min"] = min(values)
item[f"{metric}_max"] = max(values)
item["shape_stage_medians_ms"] = {}
for stage, key in (("local_shapes", "local-shapes"),
("intervals_signs", "intervals+signs"),
("surface_band", "surface-band"),
("far_parents", "far-parents")):
values = [float(shape_row[key]) for row in subset
for shape_row in row["shape_stages"] if key in shape_row]
item["shape_stage_medians_ms"][stage] = (
statistics.median(values) if values else None)
slow_per_run = [tile for row in subset for tile in row["slow_tiles"]]
item["slowest_tiles_all_runs"] = slow_per_run
summary.append(item)
(OUT / "summary.json").write_text(
json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n",
encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,154 @@
#!/usr/bin/env python3
"""Interleaved L84/L85 W8 profiles for the FI/TN heavy-tile floor."""
from __future__ import annotations
import json
import re
import statistics
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
OUT = ROOT / "Saved/Linux/l85/tile-profiles-final"
DATA = ROOT / "Standalone/data/owner_world.json"
BINARIES = {
"L84": ROOT / "Saved/Linux/l84/bin/VoxelForgeCoreHeadless-l84-clang19",
"L85": ROOT / "Saved/Linux/l85/bin/VoxelForgeCoreHeadless-clang19",
}
STAMPS = {
"L84": "c433a039b0255ce3c4794bf20313b8a5a11442feeb07b6555f585688949741a5",
"L85": "2967bb8aaea5a7beac0cd0735c0c2e5cccd7b31a9622dd33395e81ff1e5f2589",
}
CAMERAS = {
1: (25.8369140625, 3.19921875, -737.0),
2: (19.166015625, -14.2001953125, -753.3740234375),
}
NAMES = {1: "FI", 2: "TN"}
EXPECTED_NEAR = {1: 1216, 2: 1344}
EXPECTED_WHOLE = {1: 2249, 2: 2233}
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
TIMELINE = re.compile(
r"^\[CpuTileTimeline\] event=finish key=\(([-0-9]+),([-0-9]+),([-0-9]+),lod([0-9]+)\) (.*)$", re.M)
STAGES = re.compile(
r"^\[CpuTileStages\] key=\(([-0-9]+),([-0-9]+),([-0-9]+),lod([0-9]+)\) (.*)$", re.M)
def run(strate: int, version: str, rep: int) -> dict:
out_dir = OUT / version.lower()
out_dir.mkdir(parents=True, exist_ok=True)
log = out_dir / f"{NAMES[strate].lower()}-w8-r{rep}.log"
camera = ",".join(f"{v:.12f}" for v in CAMERAS[strate]) + ",0,0"
command = [
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
"env", "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", "timeout", "1740s",
str(BINARIES[version]), str(DATA), "--world-assembler-prototype",
"--stream-benchmark", "--stream-pipeline", "--workers=8", "--clip-radius=3",
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
"--profile-lod-work", "--instrument-tile-drain", "--stream-scheduler-policy=3",
]
started = time.monotonic()
with log.open("w", encoding="utf-8") as stream:
result = subprocess.run(command, cwd=ROOT, stdout=stream,
stderr=subprocess.STDOUT, check=False, timeout=1740)
output = log.read_text(encoding="utf-8", errors="replace")
source, near, pipeline = STAMP.search(output), NEAR.search(output), PIPELINE.search(output)
if result.returncode or source is None or near is None or pipeline is None:
raise RuntimeError(f"profile run failed: exit={result.returncode} {log}")
if source.group(1) != STAMPS[version]:
raise RuntimeError(f"source stamp mismatch in {log}: {source.group(1)}")
if (int(near.group(1)), int(near.group(2))) != (EXPECTED_NEAR[strate], EXPECTED_NEAR[strate]):
raise RuntimeError(f"incomplete LOD0-2 in {log}")
fields = dict(FIELDS.findall(pipeline.group(1)))
done, total = (int(value) for value in fields["whole_tiles"].split("/"))
if (done, total) != (EXPECTED_WHOLE[strate], EXPECTED_WHOLE[strate]):
raise RuntimeError(f"incomplete whole view in {log}: {done}/{total}")
stages = {}
for match in STAGES.finditer(output):
key = tuple(int(value) for value in match.group(1, 2, 3, 4))
stages[key] = dict(FIELDS.findall(match.group(5)))
tiles = []
for match in TIMELINE.finditer(output):
key = tuple(int(value) for value in match.group(1, 2, 3, 4))
if key[3] > 2:
continue
stage = stages.get(key, {})
tile = dict(FIELDS.findall(match.group(5)))
tiles.append({
"key": key,
"service_ms": float(tile["service_ms"]),
"samples": int(stage.get("samples", "0")),
"mc_grid_ms": float(stage.get("mc_grid_ms", "0")),
"mc_grid_workers": int(stage.get("mc_grid_workers", "1")),
})
if len(tiles) != EXPECTED_NEAR[strate]:
raise RuntimeError(f"incomplete near tile profile in {log}: {len(tiles)}")
full = [tile for tile in tiles if tile["samples"] == 42875]
slowest = max(full, key=lambda tile: tile["service_ms"])
target = next((tile for tile in full if tile["key"] == (0, 1, -93, 0)), None)
if target is None:
raise RuntimeError(f"target junction tile missing from profile: {log}")
row = {
"version": version, "strate": strate, "name": NAMES[strate], "replicate": rep,
"workers": 8, "lod02_ready_s": float(near.group(3)),
"whole_view_ready_s": float(fields["whole_view_ready"].removesuffix("s")),
"shape_s": float(fields["shape"].removesuffix("s")),
"far_shape_s": float(fields["far_shape"].removesuffix("s")),
"full_grid_tile_count": len(full), "slowest_full_grid_tile": slowest,
"target_tile": target, "source_stamp": source.group(1),
"log": str(log.relative_to(ROOT)),
"wall_s": round(time.monotonic() - started, 3),
}
print("[L85TileProfile] " + json.dumps(row, sort_keys=True), flush=True)
return row
def main() -> int:
rows = []
for strate in (1, 2):
for rep in range(1, 4):
order = ("L84", "L85") if rep % 2 else ("L85", "L84")
for version in order:
rows.append(run(strate, version, rep))
(OUT / "runs-partial.json").write_text(
json.dumps(rows, indent=2) + "\n", encoding="utf-8")
summary = []
for strate in (1, 2):
for version in ("L84", "L85"):
subset = [row for row in rows
if row["strate"] == strate and row["version"] == version]
item = {"strate": strate, "name": NAMES[strate], "version": version,
"runs": [row["log"] for row in subset]}
for metric in ("lod02_ready_s", "whole_view_ready_s", "shape_s", "far_shape_s"):
values = [row[metric] for row in subset]
item[f"{metric}_runs"] = values
item[f"{metric}_median"] = statistics.median(values)
for metric in ("service_ms", "mc_grid_ms"):
values = [row["slowest_full_grid_tile"][metric] for row in subset]
item[f"slowest_full_grid_{metric}_runs"] = values
item[f"slowest_full_grid_{metric}_median"] = statistics.median(values)
item["slowest_full_grid_keys"] = [row["slowest_full_grid_tile"]["key"]
for row in subset]
item["slowest_full_grid_workers"] = [
row["slowest_full_grid_tile"]["mc_grid_workers"] for row in subset]
for metric in ("service_ms", "mc_grid_ms"):
values = [row["target_tile"][metric] for row in subset]
item[f"target_{metric}_runs"] = values
item[f"target_{metric}_median"] = statistics.median(values)
item["target_grid_workers"] = [row["target_tile"]["mc_grid_workers"]
for row in subset]
item["floor_s"] = item["shape_s_median"] + item["far_shape_s_median"] \
+ item["slowest_full_grid_service_ms_median"] / 1000.0
summary.append(item)
(OUT / "summary.json").write_text(
json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n",
encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""One Unreal null-RHI streaming trace starting in the FI stratum."""
from __future__ import annotations
import json
import re
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
HOST = Path("/home/jahni/VFHost")
EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd")
OUT = ROOT / "Saved/Linux/l85/unreal-fi"
HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l85"
HOST_SAVED = HOST / "Saved/Linux/l85"
SPAWN_M = (25.8369140625, 3.19921875, -737.0)
SAMPLE_SECONDS = 0.25
MOVE_START_S = 6.0
MOVE_DURATION_S = 12.0
DWELL_S = 5.0
EXIT_S = MOVE_START_S + MOVE_DURATION_S + DWELL_S
def main() -> int:
if not EDITOR.is_file():
raise SystemExit(f"missing Unreal commandlet executable: {EDITOR}")
label = "fi-arrival-l85"
trace = OUT / f"{label}.trace.jsonl"
abslog = OUT / f"{label}.log"
stdout = OUT / f"{label}.stdout.log"
output = HOST_PLUGIN_SAVED / "output" / label
userdir = HOST_SAVED / "user" / label
zendata = HOST_SAVED / "zen" / label
ddc = HOST_SAVED / "DDC"
for directory in (OUT, output, userdir, zendata, ddc):
directory.mkdir(parents=True, exist_ok=True)
spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M)
command = [
str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended",
"-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch",
"-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer",
"-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}",
f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0",
"-archetype=TunnelNetwork", "-slot=1", "-modes=export", "-gameconfig=1",
"-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25",
"-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0",
"-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0",
f"-out={output}", "-voxel.StreamingCoarseFirst=1",
"-voxel.StreamingDynamicNearReserve=1", "-voxel.StreamingLegacyPendingResultCap=0",
"-voxel.CoreRegionConcurrentAdmission=1", "-voxel.StreamingResultBacklogCap=256",
"-voxel.StreamingReadyQueueDepth=8", "-voxel.StreamingBackgroundWorkerCap=1",
"-voxel.CoreColumnCacheDirectMapped=0", "-voxel.CoreMCGradientMemoization=1",
"-voxel.StreamingCollisionLookahead=1", "-voxel.TestMove=1",
"-voxel.TestMoveSpeedCmPerSecond=140.0",
f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}",
f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}",
f"-voxel.TestExitSeconds={EXIT_S:.1f}", "-voxel.TestMoveDirection=1,0,0",
"-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0",
"-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}",
"-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1",
"-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}",
f"-abslog={abslog}",
]
wrapped = ["taskset", "-c", "0-11", "timeout", "1740s", *command]
started = time.monotonic()
OUT.mkdir(parents=True, exist_ok=True)
with stdout.open("w", encoding="utf-8") as stream:
result = subprocess.run(wrapped, cwd=ROOT, stdout=stream,
stderr=subprocess.STDOUT, check=False, timeout=1740)
events = []
if trace.is_file():
for line in trace.read_text(encoding="utf-8", errors="replace").splitlines():
if line.startswith("{"):
try:
events.append(json.loads(line))
except json.JSONDecodeError:
pass
samples = [event for event in events if event.get("type") == "streaming_sample"]
arrival = [event for event in samples if float(event.get("t_s", 0.0)) < MOVE_START_S]
full = []
for event in arrival:
desired = sum(event.get("desired_by_lod", [0] * 9)[:3])
resident = sum(event.get("resident_by_lod", [0] * 9)[:3])
if desired and resident >= desired:
full.append(event)
break
row = {
"label": label, "exit_code": result.returncode,
"wall_s": round(time.monotonic() - started, 3),
"sample_count": len(samples), "arrival_sample_count": len(arrival),
"first_lod02_full_ready_s": full[0]["t_s"] if full else None,
"first_lod02_full_desired": (sum(full[0].get("desired_by_lod", [0] * 9)[:3])
if full else None),
"first_lod02_full_resident": (sum(full[0].get("resident_by_lod", [0] * 9)[:3])
if full else None),
"trace": str(trace.relative_to(ROOT)), "abslog": str(abslog.relative_to(ROOT)),
"stdout": str(stdout.relative_to(ROOT)),
}
failure = re.search(r"\[VoxelForgeCoreWorldTest\] failed: ([^\r\n]+)",
stdout.read_text(encoding="utf-8", errors="replace"))
if failure:
row["world_test_failure"] = failure.group(1)
(OUT / "summary.json").write_text(json.dumps(row, indent=2) + "\n",
encoding="utf-8")
print("[L85UnrealFITrace] " + json.dumps(row, sort_keys=True), flush=True)
if result.returncode or len(samples) < 40:
raise SystemExit("Unreal FI trace did not complete successfully")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+1 -1
View File
@@ -2055,7 +2055,7 @@ namespace StandaloneStreamBenchmark
FStandaloneStreamingGenerator Streamer(World,WorkerCount,bHashDensity,nullptr,MaxLevel,
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors,&Shape,bProfileColumnOverlap,
FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache,
bAllowUniformSkipsWhenHashing);
bAllowUniformSkipsWhenHashing,&Scheduler);
Streamer.SetPassageHeavyPriority(ResolvedLayout);
Streamer.SetSchedulerPolicy(SchedulerPolicy);
Streamer.SetUniformBlockProofAuditEnabled(bAuditUniformBlockProofs);
+52 -4
View File
@@ -435,6 +435,29 @@ namespace StandaloneWorldMesher
}
}
// Grid fills can be nested inside the tile worker pool. Keep at most one such fill active so a
// burst of passage tiles adds only a bounded helper group instead of multiplying the worker count.
class FStandaloneGridFillScheduler final : public VoxelForgeCore::JobScheduler
{
public:
explicit FStandaloneGridFillScheduler(const VoxelForgeCore::JobScheduler* InInner)
:Inner(InInner){}
double parallel_for(std::size_t Count,std::size_t BatchSize,std::size_t WorkerLimit,
void* Context,Task Callback,double* WorkerActiveSeconds,
std::size_t WorkerActiveCapacity) const override
{
if(!Inner)return 0.0;
std::lock_guard<std::mutex> Lock(Mutex);
return Inner->parallel_for(Count,BatchSize,WorkerLimit,Context,Callback,
WorkerActiveSeconds,WorkerActiveCapacity);
}
private:
const VoxelForgeCore::JobScheduler* Inner=nullptr;
mutable std::mutex Mutex;
};
class FStandaloneStreamingGenerator
{
private:
@@ -462,10 +485,14 @@ public:
bool bInCaptureColumnCoordinates=false,
int32 InLod8HorizonInteriorStepVoxels=0,
bool bInSharedColumnCache=true,
bool bInAllowUniformSkipsWhenHashing=false)
bool bInAllowUniformSkipsWhenHashing=false,
const VoxelForgeCore::JobScheduler* InGridFillScheduler=nullptr)
:World(&InWorld),WorkerCount(FMath::Clamp(InWorkerCount,1,16)),bHashDensity(bInHashDensity),
MeshCache(InMeshCache),MaxLevelFilter(InMaxLevelFilter),
CoreWorld(InCoreWorld),bSkipUvAndColors(bInSkipUvAndColors),bSkipMaterialGroups(bInSkipMaterialGroups),
CoreWorld(InCoreWorld),
GridFillScheduler(InGridFillScheduler
?std::make_unique<FStandaloneGridFillScheduler>(InGridFillScheduler):nullptr),
bSkipUvAndColors(bInSkipUvAndColors),bSkipMaterialGroups(bInSkipMaterialGroups),
bSkipUV(bInSkipUV),bSkipColors(bInSkipColors),
bCaptureColumnCoordinates(bInCaptureColumnCoordinates),
bSharedColumnCache(bInSharedColumnCache),
@@ -1116,6 +1143,25 @@ private:
Options.skipMaterialGroups=bSkipMaterialGroups;Options.skipUv=bSkipUV;Options.skipColors=bSkipColors;
Options.captureColumnCoordinates=bCaptureColumnCoordinates;
Options.useSharedColumnCache=bSharedColumnCache;
// Split only dense passage/mouth overlaps, and spend only worker lanes left idle by
// the tile pool. The grid fill runs on a second pool while this tile worker waits.
const uint32 PassageScore=PassageOverlapScore(Job.Key);
if(GridFillScheduler&&Job.Key.Level==0&&WorkerCount>1&&PassageScore>=9)
{
size_t ActiveTileCount=0;
{
std::lock_guard<std::mutex> Lock(Mutex);
ActiveTileCount=Running.size();
}
const int32 AvailableHelperLanes=std::max(0,
12-std::max(0,static_cast<int32>(ActiveTileCount)-1));
const int32 GridFillWorkers=std::min({4,WorkerCount,AvailableHelperLanes});
if(GridFillWorkers>1)
{
Options.gridFillScheduler=GridFillScheduler.get();
Options.gridFillWorkerLimit=static_cast<std::uint32_t>(GridFillWorkers);
}
}
Options.lod8HorizonInteriorStepVoxels=Lod8HorizonInteriorStepVoxels;
Options.profileMesherStages=(bProfileMesherStages||bInstrumentTileDrain)&&!ForceGenerate;
Options.profilePassageComposer=bProfilePassageComposer;
@@ -1330,7 +1376,7 @@ private:
const char* TileClassName=D.tileClass==VoxelForgeCore::TileClass::allAir?"air":
D.tileClass==VoxelForgeCore::TileClass::allSolid?"solid":
D.tileClass==VoxelForgeCore::TileClass::mixed?"mixed":"unknown";
std::printf("[CpuTileStages] key=(%d,%d,%d,lod%d) worker_id=%d worker_ms=%.3f payload_ms=%.3f tile_classified=%d tile_class=%s tile_skip=%d density_point_calls=%llu density_float_points=%llu density_lattice_points=%llu density_unclamped_points=%llu density_x8_points=%llu field_eval_sample_ns=%llu field_eval_sample_points=%llu samples=%llu classify_ms=%.3f column_ms=%.3f column_exclusive_ms=%.3f mc_ms=%.3f mc_grid_ms=%.3f mc_cell_ms=%.3f mc_gradient_ms=%.3f mc_interpolation_ms=%.3f mc_dedup_ms=%.3f mc_skirt_ms=%.3f mc_other_ms=%.3f connector_ms=%.3f ground_ms=%.3f cap_ms=%.3f horizon_ms=%.3f mesh_assembly_ms=%.3f candidate_cells=%llu classified_cells=%llu surface_cells=%llu proof_solid_cells=%llu proof_air_cells=%llu proof_density_samples=%llu zband_reject_cells=%llu x8_calls=%llu x8_active_lanes=%llu grad_calls=%llu grad_memo_hits=%llu grad_fallback_calls=%llu midpoint_attempts=%llu midpoint_memo_hits=%llu skirt_edges=%llu skirt_boundary_edges=%llu skirt_density_probes=%llu skirt_triangles=%u fi_local_hit=%llu fi_local_miss=%llu fi_local_conflicts=%llu fi_shared_hit=%llu fi_shared_miss=%llu fi_shared_computes=%llu fi_shared_conflicts=%llu fi_lock_wait_us=%.3f\n",
std::printf("[CpuTileStages] key=(%d,%d,%d,lod%d) worker_id=%d worker_ms=%.3f payload_ms=%.3f tile_classified=%d tile_class=%s tile_skip=%d density_point_calls=%llu density_float_points=%llu density_lattice_points=%llu density_unclamped_points=%llu density_x8_points=%llu field_eval_sample_ns=%llu field_eval_sample_points=%llu samples=%llu classify_ms=%.3f column_ms=%.3f column_exclusive_ms=%.3f mc_ms=%.3f mc_grid_ms=%.3f mc_grid_workers=%u mc_cell_ms=%.3f mc_gradient_ms=%.3f mc_interpolation_ms=%.3f mc_dedup_ms=%.3f mc_skirt_ms=%.3f mc_other_ms=%.3f connector_ms=%.3f ground_ms=%.3f cap_ms=%.3f horizon_ms=%.3f mesh_assembly_ms=%.3f candidate_cells=%llu classified_cells=%llu surface_cells=%llu proof_solid_cells=%llu proof_air_cells=%llu proof_density_samples=%llu zband_reject_cells=%llu x8_calls=%llu x8_active_lanes=%llu grad_calls=%llu grad_memo_hits=%llu grad_fallback_calls=%llu midpoint_attempts=%llu midpoint_memo_hits=%llu skirt_edges=%llu skirt_boundary_edges=%llu skirt_density_probes=%llu skirt_triangles=%u fi_local_hit=%llu fi_local_miss=%llu fi_local_conflicts=%llu fi_shared_hit=%llu fi_shared_miss=%llu fi_shared_computes=%llu fi_shared_conflicts=%llu fi_lock_wait_us=%.3f\n",
Event.Key.Coord.X,Event.Key.Coord.Y,Event.Key.Coord.Z,Event.Key.Level,
Event.WorkerId,Event.WorkerMilliseconds,Event.PayloadBuildMilliseconds,
D.tileWasClassified?1:0,TileClassName,D.meshingSkipped?1:0,
@@ -1344,7 +1390,8 @@ private:
static_cast<unsigned long long>(Event.DensitySamples),D.classificationMilliseconds,
D.columnComputeMilliseconds,D.columnEvaluationExclusiveMilliseconds,
D.marchingCubesExclusiveMilliseconds+D.marchingCubesSkirtMilliseconds,
D.marchingCubesDensityGridMilliseconds,D.marchingCubesCellClassificationMilliseconds,
D.marchingCubesDensityGridMilliseconds,D.marchingCubesGridFillWorkers,
D.marchingCubesCellClassificationMilliseconds,
D.marchingCubesGradientNormalsMilliseconds,D.marchingCubesEdgeInterpolationMilliseconds,
D.marchingCubesVertexDedupStreamMilliseconds,D.marchingCubesSkirtMilliseconds,
D.marchingCubesOtherMilliseconds,D.farConnectorShellMilliseconds,
@@ -1441,6 +1488,7 @@ private:
const FStandaloneMeshCache* MeshCache=nullptr;
int32 MaxLevelFilter=-1;
VoxelForgeCore::World* CoreWorld=nullptr;
std::unique_ptr<FStandaloneGridFillScheduler> GridFillScheduler;
bool bSkipUvAndColors=false;
bool bSkipMaterialGroups=false;
bool bSkipUV=false;
@@ -17,6 +17,9 @@
#include "HAL/IConsoleManager.h"
#include "Misc/CommandLine.h"
#include <atomic>
#include <array>
#include <algorithm>
#include <iterator>
#include <bit>
#include <cstdlib>
#include <immintrin.h>
@@ -24,6 +27,69 @@
namespace
{
struct FVoxelMesherThreadState
{
int32 OctaveBias=0,SampleStep=1,TileCells=0,TileReachBlocks=0,TileReachBlockIndex=INDEX_NONE;
FIntVector TileOrigin=FIntVector::ZeroValue;
uint8 TilePostReachFlags=VoxelGenLOD::AllTileReach;
bool bTilePostReachDebug=false,bTilePostReachBypass=false;
bool bTileReachDiagnosticTile=false,bTileCoreReachProofEnabled=false;
bool bTileCoreReachDecisionRecorded=false,bTileBlockReachValid=false;
float TileReachScale=1.0f;
std::array<uint8,VoxelGenLOD::TileReachMaxBlockCount> TileBlockReachFlags{};
static FVoxelMesherThreadState Capture()
{
FVoxelMesherThreadState State;
State.OctaveBias=VoxelGenLOD::OctaveBias;
State.SampleStep=VoxelGenLOD::SampleStep;
State.TileOrigin=VoxelGenLOD::TileOriginVoxels;
State.TileCells=VoxelGenLOD::TileCellsPerAxis;
State.TilePostReachFlags=VoxelGenLOD::TilePostReachFlags;
State.bTilePostReachDebug=VoxelGenLOD::bTilePostReachDebug;
State.bTilePostReachBypass=VoxelGenLOD::bTilePostReachBypass;
State.bTileReachDiagnosticTile=VoxelGenLOD::bTileReachDiagnosticTile;
State.bTileCoreReachProofEnabled=VoxelGenLOD::bTileCoreReachProofEnabled;
State.bTileCoreReachDecisionRecorded=VoxelGenLOD::bTileCoreReachDecisionRecorded;
State.bTileBlockReachValid=VoxelGenLOD::bTileBlockReachValid;
State.TileReachScale=VoxelGenLOD::TileReachScale;
State.TileReachBlocks=VoxelGenLOD::TileReachBlocksPerAxis;
State.TileReachBlockIndex=VoxelGenLOD::TileReachBlockIndex;
std::copy(std::begin(VoxelGenLOD::TileBlockReachFlags),
std::end(VoxelGenLOD::TileBlockReachFlags),State.TileBlockReachFlags.begin());
return State;
}
void Apply() const
{
VoxelGenLOD::OctaveBias=OctaveBias;
VoxelGenLOD::SampleStep=SampleStep;
VoxelGenLOD::TileOriginVoxels=TileOrigin;
VoxelGenLOD::TileCellsPerAxis=TileCells;
VoxelGenLOD::TilePostReachFlags=TilePostReachFlags;
VoxelGenLOD::bTilePostReachDebug=bTilePostReachDebug;
VoxelGenLOD::bTilePostReachBypass=bTilePostReachBypass;
VoxelGenLOD::bTileReachDiagnosticTile=bTileReachDiagnosticTile;
VoxelGenLOD::bTileCoreReachProofEnabled=bTileCoreReachProofEnabled;
VoxelGenLOD::bTileCoreReachDecisionRecorded=bTileCoreReachDecisionRecorded;
VoxelGenLOD::bTileBlockReachValid=bTileBlockReachValid;
VoxelGenLOD::TileReachScale=TileReachScale;
VoxelGenLOD::TileReachBlocksPerAxis=TileReachBlocks;
VoxelGenLOD::TileReachBlockIndex=TileReachBlockIndex;
std::copy(TileBlockReachFlags.begin(),TileBlockReachFlags.end(),
std::begin(VoxelGenLOD::TileBlockReachFlags));
}
};
struct FVoxelMesherThreadStateScope
{
explicit FVoxelMesherThreadStateScope(const FVoxelMesherThreadState& InTarget)
:Previous(FVoxelMesherThreadState::Capture())
{ InTarget.Apply(); }
~FVoxelMesherThreadStateScope(){Previous.Apply();}
FVoxelMesherThreadState Previous;
};
int32 GVoxelForgeUseOperatorBlock = 1;
FAutoConsoleVariableRef CVarVoxelForgeUseOperatorBlock(
TEXT("voxel.UseOperatorBlock"),
@@ -957,6 +1023,7 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
// capacité, donc plus de malloc/free de ~170 Ko (35³ floats) par tuile.
static thread_local TArray<float> DensityGrid;
DensityGrid.SetNumUninitialized(MDim * MDim * MDim);
float* const DensityGridData=DensityGrid.GetData();
const FVoxelSharedDensityGrid* ReuseGrid = InSharedDensityGrid != nullptr
? InSharedDensityGrid : SharedDensityGrid;
const int32 SharedDeltaX = ReuseGrid
@@ -1063,14 +1130,7 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
const bool bUseOperatorBlock = !bUseSharedDensityGrid
&& !bUseParameterizedBrickEvaluator
&& GVoxelForgeUseOperatorBlock != 0;
if (bUseOperatorBlock)
{
// The grid includes one sample of halo on every side. The generator groups these exact
// lattice points by chunk key and lets the prepared op stack fill each group once.
Generator->BeginDensityBlock(
OriginVoxels + FIntVector(-Step, -Step, GzLo * Step),
Step, MDim, MDim, GzHi - GzLo + 1);
}
bool bGridFillParallel=false;
if (bUseParameterizedBrickEvaluator)
{
#if WITH_EDITOR
@@ -1238,7 +1298,9 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
* (VoxelGenLOD::IsUseFusedEvaluatorEnabled() ? 4 : 1);
const auto XGroups = BuildAxisGroups(OriginVoxels.X,RoomCacheRegionSize);
const auto YGroups = BuildAxisGroups(OriginVoxels.Y,RoomCacheRegionSize);
auto SampleGridPoint = [&](int32 gx, int32 gy, int32 gz)
auto SampleGridPoint = [&](int32 gx, int32 gy, int32 gz,
FVoxelMarchingCubesStageTimings* StageTimings,
int64* DensitySampleCount,float* GridData)
{
const int32 LocalIndex = ((gz + 1) * MDim + (gy + 1)) * MDim + (gx + 1);
if(bSkipProofBlockGrid&&RequiredGridSamples[LocalIndex]==0)return;
@@ -1247,7 +1309,7 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
const int32 SharedX = SharedDeltaX + gx;
const int32 SharedY = SharedDeltaY + gy;
const int32 SharedZ = SharedDeltaZ + gz;
DensityGrid[LocalIndex] = ReuseGrid->Samples[
GridData[LocalIndex] = ReuseGrid->Samples[
(SharedZ * ReuseGrid->Dim + SharedY) * ReuseGrid->Dim + SharedX];
}
else
@@ -1272,22 +1334,22 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
bBoundarySample
?VoxelForgeCore::floating_island_detail::EIslandFieldCaller::Boundary
:VoxelForgeCore::floating_island_detail::EIslandFieldCaller::GridFill);
DensityGrid[LocalIndex] = ShapeWorldContext
GridData[LocalIndex] = ShapeWorldContext
?ShapeWorldContext->DensityAtLattice(SampleX,SampleY,SampleZ)
:Generator->GetDensityAtLattice(SampleX,SampleY,SampleZ,
bCapturePreEditGrid ? &PreEditDensity : nullptr);
if(OutStageTimings)
if(StageTimings)
{
if(bBoundarySample)++OutStageTimings->DensityGridHaloSamples;
else ++OutStageTimings->DensityGridInteriorSamples;
if(bBoundarySample)++StageTimings->DensityGridHaloSamples;
else ++StageTimings->DensityGridInteriorSamples;
}
if (bCapturePreEditGrid)
{
OutPreEditDensityGrid->Samples[LocalIndex] = PreEditDensity;
}
if (OutDensitySampleCount)
if (DensitySampleCount)
{
++(*OutDensitySampleCount);
++(*DensitySampleCount);
}
}
};
@@ -1308,7 +1370,9 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
const bool bUseShapeWorldXBatch=ActiveShapeWorld!=nullptr
&&!bUseSharedDensityGrid&&!bCapturePreEditGrid
&&(!bSkipProofBlockGrid||bStandaloneEnableActiveLaneBatching);
const auto SampleShapeWorldXBatch=[&](int32 GX0,int32 BatchCount,int32 gy,int32 gz)
const auto SampleShapeWorldXBatch=[&](int32 GX0,int32 BatchCount,int32 gy,int32 gz,
FVoxelMarchingCubesStageTimings* StageTimings,int64* DensitySampleCount,
float* GridData)
{
float Densities[8]{};
const bool bBoundarySample=GX0<0||GX0+BatchCount-1>GridDim
@@ -1320,24 +1384,24 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
:VoxelForgeCore::floating_island_detail::EIslandFieldCaller::GridFill);
ActiveShapeWorld->DensityAtLatticeX8(OriginVoxels.X+GX0*Step,Step,
OriginVoxels.Y+gy*Step,OriginVoxels.Z+gz*Step,BatchCount,Densities);
if(OutStageTimings)
if(StageTimings)
{
++OutStageTimings->X8DensityCalls;
OutStageTimings->X8ActiveLanes+=static_cast<uint64>(BatchCount);
++StageTimings->X8DensityCalls;
StageTimings->X8ActiveLanes+=static_cast<uint64>(BatchCount);
for(int32 Lane=0;Lane<BatchCount;++Lane)
{
const int32 SampleGX=GX0+Lane;
if(SampleGX<0||SampleGX>GridDim||gy<0||gy>GridDim
||gz<0||gz>GridDim)
++OutStageTimings->DensityGridHaloSamples;
else ++OutStageTimings->DensityGridInteriorSamples;
++StageTimings->DensityGridHaloSamples;
else ++StageTimings->DensityGridInteriorSamples;
}
}
for(int32 Lane=0;Lane<BatchCount;++Lane)
{
const int32 GX=GX0+Lane;
const int32 LocalIndex=((gz+1)*MDim+(gy+1))*MDim+(GX+1);
DensityGrid[LocalIndex]=Densities[Lane];
GridData[LocalIndex]=Densities[Lane];
}
const int32 LastGX=GX0+BatchCount-1;
const int32 ReachBlockX=FMath::Clamp(LastGX,0,CellsPerAxis-1)
@@ -1348,9 +1412,202 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
/VoxelGenLOD::TileReachBlockCells;
VoxelGenLOD::TileReachBlockIndex=
((ReachBlockZ*ReachBlocksPerAxis)+ReachBlockY)*ReachBlocksPerAxis+ReachBlockX;
if(OutDensitySampleCount)*OutDensitySampleCount+=BatchCount;
if(DensitySampleCount)*DensitySampleCount+=BatchCount;
};
if(bUseShapeWorldZBatch&&!bUseVerticalCacheTraversal)
const int32 GridZSampleCount=GzHi-GzLo+1;
const int32 GridYSampleCount=GridDim+2;
const bool bParallelizeGridRows=bUseShapeWorldZBatch
||bUseVerticalCacheTraversal||!bUseShapeWorldXBatch;
const std::size_t GridFillLaneCount=std::min<std::size_t>({
GridFillExecution.WorkerLimit,16,
static_cast<std::size_t>(FMath::Max(0,
bParallelizeGridRows?GridYSampleCount:GridZSampleCount))});
const bool bUseParallelShapeWorldGrid=GridFillExecution.ParallelFor!=nullptr
&&GridFillLaneCount>1&&GridFillExecution.BeginWorkerContext!=nullptr
&&GridFillExecution.EndWorkerContext!=nullptr
&&!bUseSharedDensityGrid&&!bCapturePreEditGrid&&!bSkipProofBlockGrid
&&OutCaptureGrid==nullptr;
if(bUseOperatorBlock&&!bUseParallelShapeWorldGrid)
{
// The grid includes one sample of halo on every side. The prepared operator block
// is thread-local, so a parallel fill below opens one block per worker slab.
Generator->BeginDensityBlock(
OriginVoxels+FIntVector(-Step,-Step,GzLo*Step),
Step,MDim,MDim,GzHi-GzLo+1);
}
if(bUseParallelShapeWorldGrid)
{
bGridFillParallel=true;
const FVoxelMesherThreadState TileThreadState=FVoxelMesherThreadState::Capture();
std::array<int64,16> LaneSampleCounts{};
std::array<FVoxelMarchingCubesStageTimings,16> LaneTimings{};
std::atomic<bool> bAborted{false};
auto RunLane=[&](std::size_t Lane,std::size_t Worker)
{
void* WorkerState=GridFillExecution.BeginWorkerContext(
GridFillExecution.WorkerContext,Worker);
int64& LaneSamples=LaneSampleCounts[Lane];
FVoxelMarchingCubesStageTimings& StageTimings=LaneTimings[Lane];
{
FVoxelMesherThreadStateScope ThreadStateScope(TileThreadState);
if(bUseOperatorBlock)
Generator->BeginDensityBlock(
OriginVoxels+FIntVector(-Step,-Step,GzLo*Step),Step,
MDim,MDim,GridZSampleCount);
if(bUseShapeWorldZBatch)
{
const int32 FirstGy=-1+static_cast<int32>(
(static_cast<std::size_t>(GridYSampleCount)*Lane)/GridFillLaneCount);
const int32 EndGy=-1+static_cast<int32>(
(static_cast<std::size_t>(GridYSampleCount)*(Lane+1))/GridFillLaneCount);
for(int32 gy=FirstGy;gy<EndGy;++gy)
{
if(ShouldAbortWork())
{bAborted.store(true,std::memory_order_relaxed);break;}
for(const std::vector<int32>& XGroup:XGroups)
for(const int32 gx:XGroup)
{
if(ShouldAbortWork())
{
bAborted.store(true,std::memory_order_relaxed);
break;
}
float Densities[CHUNK_SIZE+3]{};
const bool bBoundarySample=gx<0||gx>GridDim
||gy<0||gy>GridDim||GzLo<0||GzHi>GridDim;
VoxelForgeCore::Linux77Profiler::FScopedHalo l77Halo(
bBoundarySample);
VoxelForgeCore::floating_island_detail::FScopedIslandFieldCaller
islandCaller(bBoundarySample
?VoxelForgeCore::floating_island_detail::EIslandFieldCaller::Boundary
:VoxelForgeCore::floating_island_detail::EIslandFieldCaller::GridFill);
ActiveShapeWorld->DensityAtLatticeZRun(
OriginVoxels.X+gx*Step,OriginVoxels.Y+gy*Step,
OriginVoxels.Z+GzLo*Step,Step,GridZSampleCount,Densities);
for(int32 ZLane=0;ZLane<GridZSampleCount;++ZLane)
{
const int32 gz=GzLo+ZLane;
const int32 LocalIndex=
((gz+1)*MDim+(gy+1))*MDim+(gx+1);
DensityGridData[LocalIndex]=Densities[ZLane];
if(gx<0||gx>GridDim||gy<0||gy>GridDim
||gz<0||gz>GridDim)
++StageTimings.DensityGridHaloSamples;
else ++StageTimings.DensityGridInteriorSamples;
}
LaneSamples+=GridZSampleCount;
const int32 ReachBlockX=FMath::Clamp(gx,0,CellsPerAxis-1)
/VoxelGenLOD::TileReachBlockCells;
const int32 ReachBlockY=FMath::Clamp(gy,0,CellsPerAxis-1)
/VoxelGenLOD::TileReachBlockCells;
const int32 ReachBlockZ=FMath::Clamp(GzHi,0,CellsPerAxis-1)
/VoxelGenLOD::TileReachBlockCells;
VoxelGenLOD::TileReachBlockIndex=
((ReachBlockZ*ReachBlocksPerAxis)+ReachBlockY)
*ReachBlocksPerAxis+ReachBlockX;
}
if(bAborted.load(std::memory_order_relaxed))break;
}
}
else if(bUseVerticalCacheTraversal||!bUseShapeWorldXBatch)
{
const int32 FirstGy=-1+static_cast<int32>(
(static_cast<std::size_t>(GridYSampleCount)*Lane)/GridFillLaneCount);
const int32 EndGy=-1+static_cast<int32>(
(static_cast<std::size_t>(GridYSampleCount)*(Lane+1))/GridFillLaneCount);
for(int32 gy=FirstGy;gy<EndGy;++gy)
{
if(ShouldAbortWork())
{bAborted.store(true,std::memory_order_relaxed);break;}
for(const std::vector<int32>& XGroup:XGroups)
for(const int32 gx:XGroup)
{
for(int32 gz=GzLo;gz<=GzHi;++gz)
{
if(ShouldAbortWork())
{bAborted.store(true,std::memory_order_relaxed);break;}
SampleGridPoint(gx,gy,gz,&StageTimings,&LaneSamples,
DensityGridData);
}
if(bAborted.load(std::memory_order_relaxed))break;
}
if(bAborted.load(std::memory_order_relaxed))break;
}
}
else
{
const int32 FirstGz=GzLo+static_cast<int32>(
(static_cast<std::size_t>(GridZSampleCount)*Lane)/GridFillLaneCount);
const int32 EndGz=GzLo+static_cast<int32>(
(static_cast<std::size_t>(GridZSampleCount)*(Lane+1))/GridFillLaneCount);
for(int32 gz=FirstGz;gz<EndGz;++gz)
{
if(ShouldAbortWork())
{bAborted.store(true,std::memory_order_relaxed);break;}
for(const std::vector<int32>& YGroup:YGroups)
for(const std::vector<int32>& XGroup:XGroups)
for(const int32 gy:YGroup)
{
if(ShouldAbortWork())
{bAborted.store(true,std::memory_order_relaxed);break;}
for(std::size_t Offset=0;Offset<XGroup.size();)
{
const int32 BatchCount=static_cast<int32>(std::min<std::size_t>(
8,XGroup.size()-Offset));
const int32 GX0=XGroup[Offset];
bool bContiguous=true;
for(int32 BatchLane=1;BatchLane<BatchCount;++BatchLane)
bContiguous&=XGroup[
Offset+static_cast<std::size_t>(BatchLane)]==GX0+BatchLane;
if(!bContiguous)
{
for(int32 BatchLane=0;BatchLane<BatchCount;++BatchLane)
SampleGridPoint(XGroup[
Offset+static_cast<std::size_t>(BatchLane)],gy,gz,
&StageTimings,&LaneSamples,DensityGridData);
}
else
SampleShapeWorldXBatch(GX0,BatchCount,gy,gz,
&StageTimings,&LaneSamples,DensityGridData);
Offset+=static_cast<std::size_t>(BatchCount);
}
}
if(bAborted.load(std::memory_order_relaxed))break;
}
}
if(bUseOperatorBlock)Generator->EndDensityBlock();
}
GridFillExecution.EndWorkerContext(
GridFillExecution.WorkerContext,WorkerState,Worker);
};
struct FDispatchContext{decltype(RunLane)* Run=nullptr;};
FDispatchContext DispatchContext{&RunLane};
const auto DispatchLane=[](void* RawContext,std::size_t Lane,std::size_t Worker)
{
auto& Context=*static_cast<FDispatchContext*>(RawContext);
(*Context.Run)(Lane,Worker);
};
GridFillExecution.ParallelFor(GridFillExecution.SchedulerContext,
GridFillLaneCount,GridFillLaneCount,&DispatchContext,DispatchLane);
if(OutDensitySampleCount)
for(std::size_t Lane=0;Lane<GridFillLaneCount;++Lane)
*OutDensitySampleCount+=LaneSampleCounts[Lane];
if(OutStageTimings)
{
OutStageTimings->GridFillWorkerCount=static_cast<uint32>(GridFillLaneCount);
for(std::size_t Lane=0;Lane<GridFillLaneCount;++Lane)
{
OutStageTimings->DensityGridInteriorSamples+=
LaneTimings[Lane].DensityGridInteriorSamples;
OutStageTimings->DensityGridHaloSamples+=
LaneTimings[Lane].DensityGridHaloSamples;
OutStageTimings->X8DensityCalls+=LaneTimings[Lane].X8DensityCalls;
OutStageTimings->X8ActiveLanes+=LaneTimings[Lane].X8ActiveLanes;
}
}
if(bAborted.load(std::memory_order_relaxed))return FVoxelMeshData();
}
else if(bUseShapeWorldZBatch&&!bUseVerticalCacheTraversal)
{
for(const std::vector<int32>& YGroup:YGroups)
for(const std::vector<int32>& XGroup:XGroups)
@@ -1419,7 +1676,8 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
if (bUseOperatorBlock) { Generator->EndDensityBlock(); }
return FVoxelMeshData();
}
SampleGridPoint(gx,gy,gz);
SampleGridPoint(gx,gy,gz,OutStageTimings,OutDensitySampleCount,
DensityGridData);
}
}
}
@@ -1460,8 +1718,11 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
if(RequiredGridSamples[NextIndex]==0)break;
++BatchCount;
}
if(BatchCount<2)SampleGridPoint(GX0,gy,gz);
else SampleShapeWorldXBatch(GX0,BatchCount,gy,gz);
if(BatchCount<2)SampleGridPoint(
GX0,gy,gz,OutStageTimings,OutDensitySampleCount,
DensityGridData);
else SampleShapeWorldXBatch(GX0,BatchCount,gy,gz,
OutStageTimings,OutDensitySampleCount,DensityGridData);
Offset+=static_cast<size_t>(BatchCount);
continue;
}
@@ -1474,18 +1735,21 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
if(!bContiguous)
{
for(int32 Lane=0;Lane<BatchCount;++Lane)
SampleGridPoint(XGroup[Offset+static_cast<size_t>(Lane)],gy,gz);
SampleGridPoint(XGroup[Offset+static_cast<size_t>(Lane)],gy,gz,
OutStageTimings,OutDensitySampleCount,DensityGridData);
Offset+=static_cast<size_t>(BatchCount);
continue;
}
SampleShapeWorldXBatch(GX0,BatchCount,gy,gz);
SampleShapeWorldXBatch(GX0,BatchCount,gy,gz,
OutStageTimings,OutDensitySampleCount,DensityGridData);
Offset+=static_cast<size_t>(BatchCount);
}
}
else
{
for(const int32 gx:XGroup)SampleGridPoint(gx,gy,gz);
for(const int32 gx:XGroup)SampleGridPoint(
gx,gy,gz,OutStageTimings,OutDensitySampleCount,DensityGridData);
}
}
}
@@ -1506,7 +1770,7 @@ FVoxelMeshData UVoxelMarchingCubesMesher::GenerateMesh(FIntVector OriginVoxels,
GridSampleCount * sizeof(float));
}
}
if (bUseOperatorBlock)
if (bUseOperatorBlock&&!bGridFillParallel)
{
Generator->EndDensityBlock();
}
@@ -13,6 +13,7 @@
#include "CoreMinimal.h"
#include <atomic>
#include <cstddef>
#include <vector>
#include "VoxelTypes.h" // Pour FVoxelMeshData, CHUNK_SIZE, VOXEL_SIZE, etc.
#include "VoxelGenerator.h"
@@ -41,10 +42,29 @@ struct FVoxelSharedDensityGrid
}
};
// Host-owned scheduling hooks for a predicted-heavy density-grid fill. Each task owns one
// deterministic Z slab and gets the source tile's worker-local sampling context.
struct FVoxelGridFillExecution
{
using FTask = void(*)(void* taskContext, std::size_t index, std::size_t workerIndex);
using FParallelFor = double(*)(void* schedulerContext, std::size_t count,
std::size_t workerLimit, void* taskContext, FTask task);
using FBeginWorkerContext = void*(*)(void* context, std::size_t workerIndex);
using FEndWorkerContext = void(*)(void* context, void* workerState,
std::size_t workerIndex);
void* SchedulerContext = nullptr;
FParallelFor ParallelFor = nullptr;
std::size_t WorkerLimit = 1;
void* WorkerContext = nullptr;
FBeginWorkerContext BeginWorkerContext = nullptr;
FEndWorkerContext EndWorkerContext = nullptr;
};
// Optional, sampled timing output for the research audit; null in production calls.
// Mesures echantillonnees facultatives pour l'audit; pointeur nul en production.
struct FVoxelMarchingCubesStageTimings
{
uint32 GridFillWorkerCount = 1;
double DensityGridMilliseconds = 0.0;
double CellClassificationMilliseconds = 0.0;
double GradientNormalsMilliseconds = 0.0;
@@ -161,6 +181,8 @@ public:
// GenerateMesh calls finish. A malformed/incomplete lattice is ignored and falls back to
// the canonical generator path rather than changing geometry.
void SetSharedDensityGrid(const FVoxelSharedDensityGrid* InGrid) { SharedDensityGrid = InGrid; }
void SetGridFillExecution(const FVoxelGridFillExecution& InExecution)
{ GridFillExecution = InExecution; }
// Standalone's interval world is an explicit immutable sampling context. Game builds do not
// set this pointer and continue through Generator.
@@ -189,6 +211,7 @@ private:
const std::atomic<bool>* ShutdownFlag = nullptr;
const FVoxelSharedDensityGrid* SharedDensityGrid = nullptr;
FVoxelGridFillExecution GridFillExecution;
const FStandaloneShapeWorld* ShapeWorldContext = nullptr;
TArray<FVoxelHybridSharpFeatureBox> HybridSharpFeatures;
bool bUseHybridSharpMesher = false;