streaming: incremental overlay materialization and same-center priority merge (L166)

Persistent Core/overlay/candidate sets updated from deltas; same-center priority order merges changed keys (full sort
only on center change). Set and order equality audits 28/28 + 28/28, injected mismatch fails. 30 m/s UE, 7 pairs vs
9c70053: p99 10.52 -> 7.42 ms (all pairs), worst tick 12.98 -> 11.65 ms (6/7), near 42.0 -> 43.7% and forward cone
14.8 -> 16.1% (all pairs), far holes 3.20 -> 2.74% median (4/7), arrival flat. Source/: VoxelWorld.cpp, VoxelWorld.h.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed 2026-11-02 13:00:55 +01:00
1 parent 9c70053543
commit c6357b573f
4 files changed
+1290 -32

No files matched your search

+537 -32
View File
@@ -397,6 +397,12 @@ namespace
GVoxelForgeStreamingIncrementalDesiredAuditEvery, GVoxelForgeStreamingIncrementalDesiredAuditEvery,
TEXT("Compare the incremental desired set with a full rebuild every N center crossings; 0 disables the debug audit.")); TEXT("Compare the incremental desired set with a full rebuild every N center crossings; 0 disables the debug audit."));
int32 GVoxelForgeStreamingIncrementalDesiredMaterializationAuditEvery = 0;
FAutoConsoleVariableRef CVarVoxelForgeStreamingIncrementalDesiredMaterializationAuditEvery(
TEXT("voxel.StreamingIncrementalDesiredMaterializationAuditEvery"),
GVoxelForgeStreamingIncrementalDesiredMaterializationAuditEvery,
TEXT("Compare reused same-center desired materialization and priority order with a full rebuild every N cache hits; 0 disables this debug audit."));
int32 GVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch = 0; int32 GVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch = 0;
FAutoConsoleVariableRef CVarVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch( FAutoConsoleVariableRef CVarVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch(
TEXT("voxel.StreamingIncrementalDesiredAuditInjectMismatch"), TEXT("voxel.StreamingIncrementalDesiredAuditInjectMismatch"),
@@ -1183,7 +1189,17 @@ void AVoxelWorld::RegenerateAllChunks()
CachedCoreDesiredTileSet.Reset(); CachedCoreDesiredTileSet.Reset();
CachedCoreDesiredCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32); CachedCoreDesiredCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
bHasCachedCoreDesiredTiles = false; bHasCachedCoreDesiredTiles = false;
CachedDesiredCandidateKeys.Reset();
CachedDesiredCandidateSet.Reset();
CachedDesiredBaseVisualTiles.Reset();
CachedDesiredOverlayTiles.Reset();
CachedDesiredMaterializationCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
bHasCachedDesiredMaterialization = false;
CachedDesiredPriorityOrder.Reset();
CachedDesiredPriorityCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
bHasCachedDesiredPriorityOrder = false;
IncrementalDesiredCenterCrossings = 0; IncrementalDesiredCenterCrossings = 0;
IncrementalDesiredMaterializationUpdates = 0;
++DesiredEpoch; ++DesiredEpoch;
TransitionHold.Reset(); TransitionHold.Reset();
TransitionHoldQueue.Reset(); TransitionHoldQueue.Reset();
@@ -2351,6 +2367,13 @@ void AVoxelWorld::BeginPlay()
GVoxelForgeStreamingIncrementalDesiredAuditEvery = FMath::Clamp( GVoxelForgeStreamingIncrementalDesiredAuditEvery = FMath::Clamp(
GVoxelForgeStreamingIncrementalDesiredAuditEvery, 0, 1000000); GVoxelForgeStreamingIncrementalDesiredAuditEvery, 0, 1000000);
} }
if (FParse::Value(FCommandLine::Get(),
TEXT("voxel.StreamingIncrementalDesiredMaterializationAuditEvery="),
GVoxelForgeStreamingIncrementalDesiredMaterializationAuditEvery))
{
GVoxelForgeStreamingIncrementalDesiredMaterializationAuditEvery = FMath::Clamp(
GVoxelForgeStreamingIncrementalDesiredMaterializationAuditEvery, 0, 1000000);
}
if (FParse::Value(FCommandLine::Get(), if (FParse::Value(FCommandLine::Get(),
TEXT("voxel.StreamingIncrementalDesiredAuditInjectMismatch="), TEXT("voxel.StreamingIncrementalDesiredAuditInjectMismatch="),
GVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch)) GVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch))
@@ -8163,7 +8186,17 @@ bool AVoxelWorld::InitializeVoxelForgeCoreWorld(FString& OutFailureReason)
CachedCoreDesiredTileSet.Reset(); CachedCoreDesiredTileSet.Reset();
bHasCachedCoreDesiredTiles = false; bHasCachedCoreDesiredTiles = false;
CachedCoreDesiredCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32); CachedCoreDesiredCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
CachedDesiredCandidateKeys.Reset();
CachedDesiredCandidateSet.Reset();
CachedDesiredBaseVisualTiles.Reset();
CachedDesiredOverlayTiles.Reset();
CachedDesiredMaterializationCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
bHasCachedDesiredMaterialization = false;
CachedDesiredPriorityOrder.Reset();
CachedDesiredPriorityCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
bHasCachedDesiredPriorityOrder = false;
IncrementalDesiredCenterCrossings = 0; IncrementalDesiredCenterCrossings = 0;
IncrementalDesiredMaterializationUpdates = 0;
State->bSnapshotReady.store(true, std::memory_order_release); State->bSnapshotReady.store(true, std::memory_order_release);
ActiveTaskCount.fetch_add(1, std::memory_order_relaxed); ActiveTaskCount.fetch_add(1, std::memory_order_relaxed);
const double CoreCompileLaunchSeconds = bTraceCoreStartup const double CoreCompileLaunchSeconds = bTraceCoreStartup
@@ -11111,6 +11144,9 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
TArray<FVoxelTileKey>& OutLeavers) TArray<FVoxelTileKey>& OutLeavers)
{ {
TRACE_CPUPROFILER_EVENT_SCOPE(VoxelForge_BuildCoreDesiredTiles); TRACE_CPUPROFILER_EVENT_SCOPE(VoxelForge_BuildCoreDesiredTiles);
const bool bTraceDesiredBreakdown = VoxelForgeStartupTrace::IsActive();
const double DesiredBreakdownStartSeconds = bTraceDesiredBreakdown
? FPlatformTime::Seconds() : 0.0;
// Capture the resolved floor before rebuilding CollisionDesiredTiles. The gate request set // Capture the resolved floor before rebuilding CollisionDesiredTiles. The gate request set
// drops keys as soon as they have a body, while the pawn-radius set may not include a floor // drops keys as soon as they have a body, while the pawn-radius set may not include a floor
// tile below a known-empty band. Keeping the active support here prevents that body from // tile below a known-empty band. Keeping the active support here prevents that body from
@@ -11145,6 +11181,14 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
TArray<FVoxelTileKey> IncrementalCoreEntering; TArray<FVoxelTileKey> IncrementalCoreEntering;
TArray<FVoxelTileKey> IncrementalCoreLeaving; TArray<FVoxelTileKey> IncrementalCoreLeaving;
double CoreDesiredQueryMilliseconds = 0.0; double CoreDesiredQueryMilliseconds = 0.0;
double CoreDesiredPreparationMilliseconds = 0.0;
double CoreDesiredDeltaMilliseconds = 0.0;
double CoreDesiredFullFallbackMilliseconds = 0.0;
double OverlayMaterializationMilliseconds = 0.0;
double PrioritySortMilliseconds = 0.0;
double CriticalAdmissionOrderMilliseconds = 0.0;
int32 IncrementalChangedLod4PlusKeyCount = 0;
int32 IncrementalFallbackReason = 0;
bool bCoreDesiredCacheHit = false; bool bCoreDesiredCacheHit = false;
bool bCoreDesiredIncrementalDelta = false; bool bCoreDesiredIncrementalDelta = false;
bool bCoreDesiredFullFallback = false; bool bCoreDesiredFullFallback = false;
@@ -11161,6 +11205,9 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
&& DesiredAuditInterval > 0 && DesiredAuditInterval > 0
&& (IncrementalDesiredCenterCrossings % DesiredAuditInterval) == 0; && (IncrementalDesiredCenterCrossings % DesiredAuditInterval) == 0;
const double CoreDesiredPreparationStartSeconds = bTraceDesiredBreakdown
? FPlatformTime::Seconds() : 0.0;
double CoreDesiredDeltaStartSeconds = 0.0;
if (bIncrementalDesiredSet && bHasCachedCoreDesiredTiles if (bIncrementalDesiredSet && bHasCachedCoreDesiredTiles
&& CachedCoreDesiredCenter == Center) && CachedCoreDesiredCenter == Center)
{ {
@@ -11175,6 +11222,8 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
+ FMath::Abs(static_cast<int64>(Center.Z) - CachedCoreDesiredCenter.Z); + FMath::Abs(static_cast<int64>(Center.Z) - CachedCoreDesiredCenter.Z);
if (CenterDelta == 1) if (CenterDelta == 1)
{ {
CoreDesiredDeltaStartSeconds = bTraceDesiredBreakdown
? FPlatformTime::Seconds() : 0.0;
TArray<FVoxelTileKey> RingEntering; TArray<FVoxelTileKey> RingEntering;
TArray<FVoxelTileKey> RingLeaving; TArray<FVoxelTileKey> RingLeaving;
{ {
@@ -11219,11 +11268,15 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
bool bTouchesCoreSurfaceOverlay = false; bool bTouchesCoreSurfaceOverlay = false;
for (const FVoxelTileKey& Key : RingEntering) for (const FVoxelTileKey& Key : RingEntering)
{ {
bTouchesCoreSurfaceOverlay |= Key.Level >= 4; const bool bTouchesSurfaceOverlay = Key.Level >= 4;
bTouchesCoreSurfaceOverlay |= bTouchesSurfaceOverlay;
IncrementalChangedLod4PlusKeyCount += bTouchesSurfaceOverlay ? 1 : 0;
} }
for (const FVoxelTileKey& Key : RingLeaving) for (const FVoxelTileKey& Key : RingLeaving)
{ {
bTouchesCoreSurfaceOverlay |= Key.Level >= 4; const bool bTouchesSurfaceOverlay = Key.Level >= 4;
bTouchesCoreSurfaceOverlay |= bTouchesSurfaceOverlay;
IncrementalChangedLod4PlusKeyCount += bTouchesSurfaceOverlay ? 1 : 0;
} }
if (!bTouchesCoreSurfaceOverlay) if (!bTouchesCoreSurfaceOverlay)
{ {
@@ -11256,11 +11309,18 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
else else
{ {
bCoreDesiredFullFallback = true; bCoreDesiredFullFallback = true;
IncrementalFallbackReason = 1;
}
if (CoreDesiredDeltaStartSeconds != 0.0)
{
CoreDesiredDeltaMilliseconds =
(FPlatformTime::Seconds() - CoreDesiredDeltaStartSeconds) * 1000.0;
} }
} }
else else
{ {
bCoreDesiredFullFallback = true; bCoreDesiredFullFallback = true;
IncrementalFallbackReason = 2;
} }
} }
@@ -11306,21 +11366,49 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
FPlatformTime::Cycles64() - QueryStartCycles) * 1000.0; FPlatformTime::Cycles64() - QueryStartCycles) * 1000.0;
} }
} }
if (CoreDesiredPreparationStartSeconds != 0.0)
{
CoreDesiredPreparationMilliseconds =
(FPlatformTime::Seconds() - CoreDesiredPreparationStartSeconds) * 1000.0;
if (bCoreDesiredFullFallback)
{
CoreDesiredFullFallbackMilliseconds = CoreDesiredPreparationMilliseconds;
}
}
if (VoxelForgeStartupTrace::IsActive()) if (VoxelForgeStartupTrace::IsActive())
{ {
VoxelForgeStartupTrace::RecordEvent(TEXT("core_desired_cache"), FString::Printf( VoxelForgeStartupTrace::RecordEvent(TEXT("core_desired_cache"), FString::Printf(
TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"hit\":%d," TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"hit\":%d,"
"\"incremental_delta\":%d,\"full_fallback\":%d," "\"incremental_delta\":%d,\"full_fallback\":%d,"
"\"fallback_reason\":%d,\"changed_lod4plus\":%d,"
"\"crossing_count\":%d,\"audit_due\":%d,\"keys\":%d," "\"crossing_count\":%d,\"audit_due\":%d,\"keys\":%d,"
"\"delta_entering\":%d,\"delta_leaving\":%d,\"query_ms\":%s"), "\"delta_entering\":%d,\"delta_leaving\":%d,\"query_ms\":%s,"
"\"prepare_ms\":%s,\"delta_ms\":%s,\"fallback_ms\":%s"),
static_cast<unsigned long long>(GFrameCounter), Center.X, Center.Y, Center.Z, static_cast<unsigned long long>(GFrameCounter), Center.X, Center.Y, Center.Z,
bCoreDesiredCacheHit ? 1 : 0, bCoreDesiredCacheHit ? 1 : 0,
bCoreDesiredIncrementalDelta ? 1 : 0, bCoreDesiredFullFallback ? 1 : 0, bCoreDesiredIncrementalDelta ? 1 : 0, bCoreDesiredFullFallback ? 1 : 0,
IncrementalFallbackReason, IncrementalChangedLod4PlusKeyCount,
IncrementalDesiredCenterCrossings, bShouldAuditIncrementalDesired ? 1 : 0, IncrementalDesiredCenterCrossings, bShouldAuditIncrementalDesired ? 1 : 0,
CoreDesiredTiles ? CoreDesiredTiles->Num() CoreDesiredTiles ? CoreDesiredTiles->Num()
: static_cast<int32>(UncachedCoreDesiredTiles.size()), : static_cast<int32>(UncachedCoreDesiredTiles.size()),
IncrementalCoreEntering.Num(), IncrementalCoreLeaving.Num(), IncrementalCoreEntering.Num(), IncrementalCoreLeaving.Num(),
*FString::SanitizeFloat(static_cast<float>(CoreDesiredQueryMilliseconds)))); *FString::SanitizeFloat(static_cast<float>(CoreDesiredQueryMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(CoreDesiredPreparationMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(CoreDesiredDeltaMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(CoreDesiredFullFallbackMilliseconds))));
}
// Drop completed collision-gate requests before checking the materialization cache. A gate
// set that changed during this rebuild must invalidate the cached overlay union.
for (auto It = CollisionGateDesiredTiles.CreateIterator(); It; ++It)
{
const FVoxelTileKey& Key = *It;
const TCHAR* UnavailableReason = nullptr;
if (IsTileCollisionReady(Key) || CollisionNotRequiredTiles.Contains(Key)
|| IsCollisionTileUnavailable(Key, UnavailableReason))
{
It.RemoveCurrent();
}
} }
const FIntVector LookaheadHeadingStep = GetStreamingHeadingStep(PlayerHeading, PlayerPawn); const FIntVector LookaheadHeadingStep = GetStreamingHeadingStep(PlayerHeading, PlayerPawn);
@@ -11329,23 +11417,57 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
int32 LookaheadCollisionTilesAdded = 0; int32 LookaheadCollisionTilesAdded = 0;
int32 LookaheadDesiredTilesMissing = 0; int32 LookaheadDesiredTilesMissing = 0;
TArray<FVoxelTileKey> CandidateKeys; const double OverlayMaterializationStartSeconds = bTraceDesiredBreakdown
TSet<FVoxelTileKey> CandidateSet; ? FPlatformTime::Seconds() : 0.0;
TSet<FVoxelTileKey> BaseVisualTiles; const bool bIncrementalMaterializationUpdate = bIncrementalDesiredSet
&& bHasCachedCoreDesiredTiles && CoreDesiredTiles == &CachedCoreDesiredTiles
&& bHasCachedDesiredMaterialization
&& (CachedDesiredMaterializationCenter == Center || bCoreDesiredIncrementalDelta);
if (bIncrementalMaterializationUpdate)
{
++IncrementalDesiredMaterializationUpdates;
}
const int32 MaterializationAuditInterval = FMath::Max(
0, GVoxelForgeStreamingIncrementalDesiredMaterializationAuditEvery);
const bool bShouldAuditDesiredMaterialization = bIncrementalMaterializationUpdate
&& MaterializationAuditInterval > 0
&& (IncrementalDesiredMaterializationUpdates % MaterializationAuditInterval) == 0;
const bool bShouldRunFullDesiredAudit = bShouldAuditIncrementalDesired
|| bShouldAuditDesiredMaterialization;
TArray<FVoxelTileKey>& CandidateKeys = CachedDesiredCandidateKeys;
TSet<FVoxelTileKey>& CandidateSet = CachedDesiredCandidateSet;
TSet<FVoxelTileKey>& BaseVisualTiles = CachedDesiredBaseVisualTiles;
TSet<FVoxelTileKey> FullVisualAnchorTiles; TSet<FVoxelTileKey> FullVisualAnchorTiles;
TSet<FVoxelTileKey> CollisionAnchorTiles; TSet<FVoxelTileKey> CollisionAnchorTiles;
TSet<FVoxelTileKey> OverlayDesiredTiles; TSet<FVoxelTileKey>& OverlayDesiredTiles = CachedDesiredOverlayTiles;
TSet<FVoxelTileKey> CurrentOverlayDesiredTiles;
TArray<FVoxelTileKey> DesiredEnteringTiles; TArray<FVoxelTileKey> DesiredEnteringTiles;
TArray<FVoxelTileKey> PriorityEnteringTiles;
TArray<FVoxelTileKey> PriorityLeavingTiles;
bool bCandidateKeySetChanged = false;
if (!bIncrementalMaterializationUpdate)
{
CandidateKeys.Reset();
CandidateSet.Reset();
BaseVisualTiles.Reset();
OverlayDesiredTiles.Reset();
CachedDesiredPriorityOrder.Reset();
bHasCachedDesiredPriorityOrder = false;
}
auto AddCandidate = [&](const FVoxelTileKey& Key, bool bOverlay) auto AddCandidate = [&](const FVoxelTileKey& Key, bool bOverlay)
{ {
if (bOverlay) if (bOverlay)
{ {
CurrentOverlayDesiredTiles.Add(Key);
if (bIncrementalMaterializationUpdate) return;
OverlayDesiredTiles.Add(Key); OverlayDesiredTiles.Add(Key);
} }
if (!CandidateSet.Contains(Key)) if (!CandidateSet.Contains(Key))
{ {
CandidateSet.Add(Key); CandidateSet.Add(Key);
CandidateKeys.Add(Key); CandidateKeys.Add(Key);
bCandidateKeySetChanged = true;
if (bIncrementalMaterializationUpdate) PriorityEnteringTiles.Add(Key);
const uint32* PreviousStamp = DesiredStamped.Find(Key); const uint32* PreviousStamp = DesiredStamped.Find(Key);
if (bIncrementalDesiredSet if (bIncrementalDesiredSet
&& (PreviousStamp == nullptr || *PreviousStamp != PreviousDesiredStamp)) && (PreviousStamp == nullptr || *PreviousStamp != PreviousDesiredStamp))
@@ -11361,14 +11483,14 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
AddCandidate(RuntimeKey, false); AddCandidate(RuntimeKey, false);
BaseVisualTiles.Add(RuntimeKey); BaseVisualTiles.Add(RuntimeKey);
}; };
if (CoreDesiredTiles) if (!bIncrementalMaterializationUpdate && CoreDesiredTiles)
{ {
for (const FVoxelTileKey& RuntimeKey : *CoreDesiredTiles) for (const FVoxelTileKey& RuntimeKey : *CoreDesiredTiles)
{ {
AddCoreDesiredTile(RuntimeKey); AddCoreDesiredTile(RuntimeKey);
} }
} }
else else if (!bIncrementalMaterializationUpdate)
{ {
for (const VoxelForgeCore::TileKey& Key : UncachedCoreDesiredTiles) for (const VoxelForgeCore::TileKey& Key : UncachedCoreDesiredTiles)
{ {
@@ -11417,16 +11539,8 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
// Preserve unresolved support requests from the existing collision gate. They remain in Core // Preserve unresolved support requests from the existing collision gate. They remain in Core
// tile coordinates, and level-0 bounds use the Core voxel scale through GetPlayerSupportTile. // tile coordinates, and level-0 bounds use the Core voxel scale through GetPlayerSupportTile.
for (auto It = CollisionGateDesiredTiles.CreateIterator(); It; ++It) for (const FVoxelTileKey& Key : CollisionGateDesiredTiles)
{ {
const FVoxelTileKey& Key = *It;
const TCHAR* UnavailableReason = nullptr;
if (IsTileCollisionReady(Key) || CollisionNotRequiredTiles.Contains(Key)
|| IsCollisionTileUnavailable(Key, UnavailableReason))
{
It.RemoveCurrent();
continue;
}
AddCandidate(Key, true); AddCandidate(Key, true);
CollisionDesiredTiles.Add(Key); CollisionDesiredTiles.Add(Key);
CollisionAnchorTiles.Add(Key); CollisionAnchorTiles.Add(Key);
@@ -11436,13 +11550,101 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
{ {
AddCandidate(ResolvedSupportTile, true); AddCandidate(ResolvedSupportTile, true);
CollisionDesiredTiles.Add(ResolvedSupportTile); CollisionDesiredTiles.Add(ResolvedSupportTile);
if (!BaseVisualTiles.Contains(ResolvedSupportTile) const bool bResolvedSupportIsCoreDesired = bIncrementalMaterializationUpdate
? CachedCoreDesiredTileSet.Contains(ResolvedSupportTile)
: BaseVisualTiles.Contains(ResolvedSupportTile);
if (!bResolvedSupportIsCoreDesired
&& !FullVisualAnchorTiles.Contains(ResolvedSupportTile)) && !FullVisualAnchorTiles.Contains(ResolvedSupportTile))
{ {
CollisionAnchorTiles.Add(ResolvedSupportTile); CollisionAnchorTiles.Add(ResolvedSupportTile);
} }
} }
const auto AddCachedCandidateKey = [this, &CandidateKeys, &CandidateSet,
&DesiredEnteringTiles, &PriorityEnteringTiles,
&PreviousDesiredStamp,
&bCandidateKeySetChanged](
const FVoxelTileKey& Key)
{
if (!CandidateSet.Contains(Key))
{
CandidateSet.Add(Key);
CandidateKeys.Add(Key);
bCandidateKeySetChanged = true;
PriorityEnteringTiles.Add(Key);
const uint32* PreviousStamp = DesiredStamped.Find(Key);
if (GVoxelForgeStreamingIncrementalDesiredSet != 0
&& (PreviousStamp == nullptr || *PreviousStamp != PreviousDesiredStamp))
{
DesiredEnteringTiles.Add(Key);
}
}
DesiredStamped.FindOrAdd(Key) = DesiredStamp;
};
const auto RemoveCachedCandidateKey = [this, &CandidateKeys, &CandidateSet,
&PriorityLeavingTiles,
&bCandidateKeySetChanged](
const FVoxelTileKey& Key)
{
if (CandidateSet.Contains(Key))
{
CandidateSet.Remove(Key);
CandidateKeys.RemoveSingleSwap(Key, EAllowShrinking::No);
bCandidateKeySetChanged = true;
PriorityLeavingTiles.Add(Key);
}
};
if (bIncrementalMaterializationUpdate)
{
for (const FVoxelTileKey& Key : IncrementalCoreLeaving)
{
BaseVisualTiles.Remove(Key);
if (!CurrentOverlayDesiredTiles.Contains(Key))
{
RemoveCachedCandidateKey(Key);
}
}
for (const FVoxelTileKey& Key : IncrementalCoreEntering)
{
BaseVisualTiles.Add(Key);
AddCachedCandidateKey(Key);
}
for (auto It = OverlayDesiredTiles.CreateIterator(); It; ++It)
{
const FVoxelTileKey Key = *It;
if (!CurrentOverlayDesiredTiles.Contains(Key))
{
It.RemoveCurrent();
if (!BaseVisualTiles.Contains(Key))
{
RemoveCachedCandidateKey(Key);
}
}
}
for (const FVoxelTileKey& Key : CurrentOverlayDesiredTiles)
{
if (!OverlayDesiredTiles.Contains(Key))
{
OverlayDesiredTiles.Add(Key);
AddCachedCandidateKey(Key);
}
}
for (const FVoxelTileKey& Key : CandidateKeys)
{
DesiredStamped.FindOrAdd(Key) = DesiredStamp;
}
}
if (bIncrementalDesiredSet)
{
CachedDesiredMaterializationCenter = Center;
bHasCachedDesiredMaterialization = true;
}
else
{
bHasCachedDesiredMaterialization = false;
bHasCachedDesiredPriorityOrder = false;
}
const FVoxelTileKey PlayerTile(GetTileCoordAtLocalCm(WorldToLocalCm(PlayerPosition)), 0); const FVoxelTileKey PlayerTile(GetTileCoordAtLocalCm(WorldToLocalCm(PlayerPosition)), 0);
FVoxelTileKey OccupiedTile = PlayerTile; FVoxelTileKey OccupiedTile = PlayerTile;
FVoxelTileKey SupportTile; FVoxelTileKey SupportTile;
@@ -11493,8 +11695,15 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
CollisionOnlyTiles.Add(Key); CollisionOnlyTiles.Add(Key);
} }
} }
if (OverlayMaterializationStartSeconds != 0.0)
{
OverlayMaterializationMilliseconds =
(FPlatformTime::Seconds() - OverlayMaterializationStartSeconds) * 1000.0;
}
if (bShouldAuditIncrementalDesired) TSet<FVoxelTileKey> FullReferenceDesiredSetForOrder;
TArray<FVoxelTileKey> FullReferenceDesiredKeysForOrder;
if (bShouldRunFullDesiredAudit)
{ {
std::vector<VoxelForgeCore::TileKey> FullReferenceCoreTiles; std::vector<VoxelForgeCore::TileKey> FullReferenceCoreTiles;
const FVector CameraVoxels = WorldToLocalCm(PlayerPosition) / GetActiveVoxelSizeCm(); const FVector CameraVoxels = WorldToLocalCm(PlayerPosition) / GetActiveVoxelSizeCm();
@@ -11507,30 +11716,60 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
TSet<FVoxelTileKey> FullReferenceCoreSet; TSet<FVoxelTileKey> FullReferenceCoreSet;
FullReferenceCoreSet.Reserve(static_cast<int32>(FullReferenceCoreTiles.size())); FullReferenceCoreSet.Reserve(static_cast<int32>(FullReferenceCoreTiles.size()));
TSet<FVoxelTileKey> FullReferenceDesiredSet; TSet<FVoxelTileKey> FullReferenceOverlayDesiredSet;
FullReferenceDesiredSet.Reserve( for (const FVoxelStreamingAnchor& Anchor : StreamingAnchors)
static_cast<int32>(FullReferenceCoreTiles.size()) + OverlayDesiredTiles.Num()); {
if (!Anchor.Actor.IsValid()) continue;
const FIntVector AnchorCenter = Anchor.LastChunk;
const int32 XYRadius = FMath::Clamp(Anchor.XYRadiusChunks, 0, 4);
const int32 ZBelow = FMath::Clamp(Anchor.ZBelowChunks, 0, 4);
const int32 ZAbove = FMath::Clamp(Anchor.ZAboveChunks, 0, 4);
for (int32 Dz = -ZBelow; Dz <= ZAbove; ++Dz)
for (int32 Dy = -XYRadius; Dy <= XYRadius; ++Dy)
for (int32 Dx = -XYRadius; Dx <= XYRadius; ++Dx)
{
FullReferenceOverlayDesiredSet.Add(
FVoxelTileKey(AnchorCenter + FIntVector(Dx, Dy, Dz), 0));
}
}
for (const FVoxelTileKey& Key : CollisionGateDesiredTiles)
{
FullReferenceOverlayDesiredSet.Add(Key);
}
if (bRetainResolvedSupport)
{
FullReferenceOverlayDesiredSet.Add(ResolvedSupportTile);
}
FullReferenceDesiredSetForOrder.Reserve(
static_cast<int32>(FullReferenceCoreTiles.size())
+ FullReferenceOverlayDesiredSet.Num());
for (const VoxelForgeCore::TileKey& Key : FullReferenceCoreTiles) for (const VoxelForgeCore::TileKey& Key : FullReferenceCoreTiles)
{ {
const FVoxelTileKey RuntimeKey( const FVoxelTileKey RuntimeKey(
FIntVector(Key.coordinate.x, Key.coordinate.y, Key.coordinate.z), Key.lod); FIntVector(Key.coordinate.x, Key.coordinate.y, Key.coordinate.z), Key.lod);
FullReferenceCoreSet.Add(RuntimeKey); FullReferenceCoreSet.Add(RuntimeKey);
FullReferenceDesiredSet.Add(RuntimeKey); FullReferenceDesiredSetForOrder.Add(RuntimeKey);
} }
for (const FVoxelTileKey& Key : OverlayDesiredTiles) for (const FVoxelTileKey& Key : FullReferenceOverlayDesiredSet)
{ {
FullReferenceDesiredSet.Add(Key); FullReferenceDesiredSetForOrder.Add(Key);
}
FullReferenceDesiredKeysForOrder.Reserve(FullReferenceDesiredSetForOrder.Num());
for (const FVoxelTileKey& Key : FullReferenceDesiredSetForOrder)
{
FullReferenceDesiredKeysForOrder.Add(Key);
} }
int32 MissingFromIncrementalDesired = 0; int32 MissingFromIncrementalDesired = 0;
int32 ExtraInIncrementalDesired = 0; int32 ExtraInIncrementalDesired = 0;
for (const FVoxelTileKey& Key : FullReferenceDesiredSet) for (const FVoxelTileKey& Key : FullReferenceDesiredSetForOrder)
{ {
if (!CandidateSet.Contains(Key)) ++MissingFromIncrementalDesired; if (!CandidateSet.Contains(Key)) ++MissingFromIncrementalDesired;
} }
for (const FVoxelTileKey& Key : CandidateSet) for (const FVoxelTileKey& Key : CandidateSet)
{ {
if (!FullReferenceDesiredSet.Contains(Key)) ++ExtraInIncrementalDesired; if (!FullReferenceDesiredSetForOrder.Contains(Key)) ++ExtraInIncrementalDesired;
} }
TSet<FVoxelTileKey> FullReferenceCollisionOnly; TSet<FVoxelTileKey> FullReferenceCollisionOnly;
@@ -11554,7 +11793,7 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
if (GVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch != 0) if (GVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch != 0)
{ {
FullReferenceDesiredSet.Add( FullReferenceDesiredSetForOrder.Add(
FVoxelTileKey(FIntVector(MAX_int32, MAX_int32, MAX_int32), 8)); FVoxelTileKey(FIntVector(MAX_int32, MAX_int32, MAX_int32), 8));
++MissingFromIncrementalDesired; ++MissingFromIncrementalDesired;
} }
@@ -11565,11 +11804,14 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"crossing\":%d," TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"crossing\":%d,"
"\"core_keys\":%d,\"reference_keys\":%d,\"candidate_keys\":%d," "\"core_keys\":%d,\"reference_keys\":%d,\"candidate_keys\":%d,"
"\"missing\":%d,\"extra\":%d,\"collision_only_missing\":%d," "\"missing\":%d,\"extra\":%d,\"collision_only_missing\":%d,"
"\"collision_only_extra\":%d,\"pass\":%d"), "\"collision_only_extra\":%d,\"materialization_update\":%d,"
"\"materialization_audit\":%d,\"pass\":%d"),
static_cast<unsigned long long>(GFrameCounter), Center.X, Center.Y, Center.Z, static_cast<unsigned long long>(GFrameCounter), Center.X, Center.Y, Center.Z,
IncrementalDesiredCenterCrossings, static_cast<int32>(FullReferenceCoreTiles.size()), IncrementalDesiredCenterCrossings, static_cast<int32>(FullReferenceCoreTiles.size()),
FullReferenceDesiredSet.Num(), CandidateSet.Num(), MissingFromIncrementalDesired, FullReferenceDesiredSetForOrder.Num(), CandidateSet.Num(), MissingFromIncrementalDesired,
ExtraInIncrementalDesired, MissingCollisionOnly, ExtraCollisionOnly, ExtraInIncrementalDesired, MissingCollisionOnly, ExtraCollisionOnly,
bIncrementalMaterializationUpdate ? 1 : 0,
bShouldAuditDesiredMaterialization ? 1 : 0,
bDesiredSetsMatch ? 1 : 0)); bDesiredSetsMatch ? 1 : 0));
if (!bDesiredSetsMatch) if (!bDesiredSetsMatch)
{ {
@@ -11583,6 +11825,108 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
} }
} }
const double PrioritySortStartSeconds = bTraceDesiredBreakdown
? FPlatformTime::Seconds() : 0.0;
const bool bCanIncrementallyUpdateDesiredPriorityOrder = bIncrementalMaterializationUpdate
&& bHasCachedDesiredPriorityOrder
&& CachedDesiredPriorityCenter == Center
&& CachedDesiredPriorityOrder.Num()
== CandidateSet.Num() - PriorityEnteringTiles.Num() + PriorityLeavingTiles.Num();
bool bPriorityOrderUpdated = false;
bool bReuseDesiredPriorityOrder = false;
if (bCanIncrementallyUpdateDesiredPriorityOrder)
{
const auto IsPriorityBefore = [Center](const FVoxelTileKey& A, const FVoxelTileKey& B)
{
const auto Tier = [Center](const FVoxelTileKey& Key)
{
if (Key.Level == 0 && FMath::Abs(Key.Coord.X - Center.X) <= 1
&& FMath::Abs(Key.Coord.Y - Center.Y) <= 1
&& FMath::Abs(Key.Coord.Z - Center.Z) <= 1)
{
return 0;
}
return Key.Level <= 2 ? 1 : 2;
};
const int32 TierA = Tier(A);
const int32 TierB = Tier(B);
if (TierA != TierB) return TierA < TierB;
if (TierA == 1 && A.Level != B.Level) return A.Level < B.Level;
if (TierA == 2 && A.Level != B.Level) return A.Level > B.Level;
const auto DistanceSquared = [Center](const FVoxelTileKey& Key)
{
const double Scale = static_cast<double>(1 << Key.Level);
const double DX = (static_cast<double>(Key.Coord.X) + 0.5) * Scale - Center.X;
const double DY = (static_cast<double>(Key.Coord.Y) + 0.5) * Scale - Center.Y;
const double DZ = (static_cast<double>(Key.Coord.Z) + 0.5) * Scale - Center.Z;
return DX * DX + DY * DY + DZ * DZ;
};
const double DistanceA = DistanceSquared(A);
const double DistanceB = DistanceSquared(B);
if (DistanceA != DistanceB) return DistanceA < DistanceB;
if (A.Level != B.Level) return A.Level < B.Level;
if (A.Coord.Z != B.Coord.Z) return A.Coord.Z < B.Coord.Z;
if (A.Coord.Y != B.Coord.Y) return A.Coord.Y < B.Coord.Y;
return A.Coord.X < B.Coord.X;
};
if (!PriorityLeavingTiles.IsEmpty())
{
TSet<FVoxelTileKey> LeavingSet;
LeavingSet.Reserve(PriorityLeavingTiles.Num());
for (const FVoxelTileKey& Key : PriorityLeavingTiles) LeavingSet.Add(Key);
CachedDesiredPriorityOrder.RemoveAll([&LeavingSet](const FVoxelTileKey& Key)
{
return LeavingSet.Contains(Key);
});
}
if (!PriorityEnteringTiles.IsEmpty())
{
PriorityEnteringTiles.Sort(IsPriorityBefore);
TArray<FVoxelTileKey> MergedOrder;
MergedOrder.Reserve(
CachedDesiredPriorityOrder.Num() + PriorityEnteringTiles.Num());
int32 ExistingIndex = 0;
int32 EnteringIndex = 0;
while (ExistingIndex < CachedDesiredPriorityOrder.Num()
&& EnteringIndex < PriorityEnteringTiles.Num())
{
const FVoxelTileKey& ExistingKey = CachedDesiredPriorityOrder[ExistingIndex];
const FVoxelTileKey& EnteringKey = PriorityEnteringTiles[EnteringIndex];
if (ExistingKey == EnteringKey)
{
MergedOrder.Add(ExistingKey);
++ExistingIndex;
++EnteringIndex;
}
else if (IsPriorityBefore(EnteringKey, ExistingKey))
{
MergedOrder.Add(EnteringKey);
++EnteringIndex;
}
else
{
MergedOrder.Add(ExistingKey);
++ExistingIndex;
}
}
while (ExistingIndex < CachedDesiredPriorityOrder.Num())
{
MergedOrder.Add(CachedDesiredPriorityOrder[ExistingIndex++]);
}
while (EnteringIndex < PriorityEnteringTiles.Num())
{
MergedOrder.Add(PriorityEnteringTiles[EnteringIndex++]);
}
CachedDesiredPriorityOrder = MoveTemp(MergedOrder);
}
CachedDesiredPriorityCenter = Center;
bPriorityOrderUpdated = bCandidateKeySetChanged;
bReuseDesiredPriorityOrder = !bPriorityOrderUpdated;
DesiredSorted = CachedDesiredPriorityOrder;
}
else
{
std::vector<VoxelForgeCore::TileKey> PriorityInput; std::vector<VoxelForgeCore::TileKey> PriorityInput;
PriorityInput.reserve(static_cast<std::size_t>(CandidateKeys.Num())); PriorityInput.reserve(static_cast<std::size_t>(CandidateKeys.Num()));
for (const FVoxelTileKey& Key : CandidateKeys) for (const FVoxelTileKey& Key : CandidateKeys)
@@ -11597,6 +11941,18 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
DesiredSorted.Emplace( DesiredSorted.Emplace(
FIntVector(Key.coordinate.x, Key.coordinate.y, Key.coordinate.z), Key.lod); FIntVector(Key.coordinate.x, Key.coordinate.y, Key.coordinate.z), Key.lod);
} }
if (bIncrementalDesiredSet)
{
CachedDesiredPriorityOrder = DesiredSorted;
CachedDesiredPriorityCenter = Center;
bHasCachedDesiredPriorityOrder = true;
}
}
if (PrioritySortStartSeconds != 0.0)
{
PrioritySortMilliseconds =
(FPlatformTime::Seconds() - PrioritySortStartSeconds) * 1000.0;
}
RestoreKnownEmptyCollisionTiles(); RestoreKnownEmptyCollisionTiles();
ReconcileCollisionDesiredTiles(); ReconcileCollisionDesiredTiles();
@@ -11610,6 +11966,8 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
} }
CancelObsoleteTileWork(); CancelObsoleteTileWork();
const double CriticalAdmissionOrderStartSeconds = bTraceDesiredBreakdown
? FPlatformTime::Seconds() : 0.0;
auto AddCriticalTile = [this](const FVoxelTileKey& Key) auto AddCriticalTile = [this](const FVoxelTileKey& Key)
{ {
if (IsDesired(Key) && !CriticalDesiredTiles.Contains(Key)) if (IsDesired(Key) && !CriticalDesiredTiles.Contains(Key))
@@ -11663,6 +12021,120 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
DesiredSorted.Insert(Key, 0); DesiredSorted.Insert(Key, 0);
} }
} }
if (CriticalAdmissionOrderStartSeconds != 0.0)
{
CriticalAdmissionOrderMilliseconds =
(FPlatformTime::Seconds() - CriticalAdmissionOrderStartSeconds) * 1000.0;
}
if (bShouldRunFullDesiredAudit)
{
std::vector<VoxelForgeCore::TileKey> FullReferencePriorityInput;
FullReferencePriorityInput.reserve(
static_cast<std::size_t>(FullReferenceDesiredKeysForOrder.Num()));
for (const FVoxelTileKey& Key : FullReferenceDesiredKeysForOrder)
{
FullReferencePriorityInput.push_back(
{{Key.Coord.X, Key.Coord.Y, Key.Coord.Z}, Key.Level});
}
const std::vector<VoxelForgeCore::TileKey> FullReferencePriorityOrder =
VoxelForgeCore::World::prioritizeTiles(
FullReferencePriorityInput, {Center.X, Center.Y, Center.Z});
TArray<FVoxelTileKey> FullReferenceDesiredOrder;
FullReferenceDesiredOrder.Reserve(
static_cast<int32>(FullReferencePriorityOrder.size()));
for (const VoxelForgeCore::TileKey& Key : FullReferencePriorityOrder)
{
FullReferenceDesiredOrder.Emplace(
FIntVector(Key.coordinate.x, Key.coordinate.y, Key.coordinate.z), Key.lod);
}
TArray<FVoxelTileKey> FullReferenceCriticalTiles;
const auto AddReferenceCriticalTile = [&](const FVoxelTileKey& Key)
{
if (FullReferenceDesiredSetForOrder.Contains(Key)
&& !FullReferenceCriticalTiles.Contains(Key))
{
FullReferenceCriticalTiles.Add(Key);
}
};
if (CollisionGateDesiredTiles.Contains(PawnGateRequestedSupportTile))
{
AddReferenceCriticalTile(PawnGateRequestedSupportTile);
}
for (const FVoxelTileKey& Key : FullReferenceDesiredOrder)
{
if (CollisionGateDesiredTiles.Contains(Key)) AddReferenceCriticalTile(Key);
}
AddReferenceCriticalTile(OccupiedTile);
if (HeadingStep != FIntVector::ZeroValue)
{
FVoxelTileKey NextTile = OccupiedTile;
NextTile.Coord += HeadingStep;
AddReferenceCriticalTile(NextTile);
}
AddReferenceCriticalTile(PlayerTile);
for (int32 Dy = -1; Dy <= 1; ++Dy)
for (int32 Dx = -1; Dx <= 1; ++Dx)
{
const FVoxelTileKey WalkableKey(OccupiedTile.Coord + FIntVector(Dx, Dy, 0), 0);
if (CollisionDesiredTiles.Contains(WalkableKey))
{
AddReferenceCriticalTile(WalkableKey);
}
}
for (int32 Ahead = 1; Ahead <= LookaheadTiles; ++Ahead)
{
const FIntVector AheadCenter = OccupiedTile.Coord + HeadingStep * Ahead;
for (int32 Dy = -1; Dy <= 1; ++Dy)
for (int32 Dx = -1; Dx <= 1; ++Dx)
{
const FVoxelTileKey WalkableKey(AheadCenter + FIntVector(Dx, Dy, 0), 0);
if (CollisionDesiredTiles.Contains(WalkableKey))
{
AddReferenceCriticalTile(WalkableKey);
}
}
}
for (int32 Index = FullReferenceCriticalTiles.Num() - 1; Index >= 0; --Index)
{
const int32 ExistingIndex = FullReferenceDesiredOrder.Find(
FullReferenceCriticalTiles[Index]);
if (ExistingIndex > 0)
{
const FVoxelTileKey Key = FullReferenceDesiredOrder[ExistingIndex];
FullReferenceDesiredOrder.RemoveAt(ExistingIndex, 1, EAllowShrinking::No);
FullReferenceDesiredOrder.Insert(Key, 0);
}
}
int32 PriorityOrderMismatches = FMath::Abs(
FullReferenceDesiredOrder.Num() - DesiredSorted.Num());
const int32 CommonOrderLength = FMath::Min(
FullReferenceDesiredOrder.Num(), DesiredSorted.Num());
for (int32 Index = 0; Index < CommonOrderLength; ++Index)
{
if (FullReferenceDesiredOrder[Index] != DesiredSorted[Index])
{
++PriorityOrderMismatches;
}
}
const bool bPriorityOrdersMatch = PriorityOrderMismatches == 0;
VoxelForgeStartupTrace::RecordEvent(TEXT("incremental_desired_order_audit"), FString::Printf(
TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"reference_keys\":%d,"
"\"candidate_keys\":%d,\"mismatches\":%d,"
"\"materialization_update\":%d,\"pass\":%d"),
static_cast<unsigned long long>(GFrameCounter), Center.X, Center.Y, Center.Z,
FullReferenceDesiredOrder.Num(), DesiredSorted.Num(), PriorityOrderMismatches,
bIncrementalMaterializationUpdate ? 1 : 0, bPriorityOrdersMatch ? 1 : 0));
if (!bPriorityOrdersMatch)
{
UE_LOG(LogTemp, Error,
TEXT("[VoxelForgeIncrementalDesiredOrderAudit] mismatch keys=%d mismatches=%d reused=%d"),
DesiredSorted.Num(), PriorityOrderMismatches,
bReuseDesiredPriorityOrder ? 1 : 0);
FPlatformMisc::RequestExitWithStatus(
false, 1, TEXT("VoxelForge incremental desired priority-order equality failure"));
checkf(false, TEXT("VoxelForge incremental desired priority-order equality failure"));
}
}
if (bIncrementalDesiredSet && bHadPreviousDesiredSet) if (bIncrementalDesiredSet && bHadPreviousDesiredSet)
{ {
constexpr int32 MaxSupportedTileLevel = 8; constexpr int32 MaxSupportedTileLevel = 8;
@@ -11777,6 +12249,37 @@ void AVoxelWorld::BuildCoreDesiredTiles(const FIntVector& Center, const FVector&
} }
StartPendingCoreRegionCompile(); StartPendingCoreRegionCompile();
} }
if (bTraceDesiredBreakdown)
{
const double TotalMilliseconds =
(FPlatformTime::Seconds() - DesiredBreakdownStartSeconds) * 1000.0;
VoxelForgeStartupTrace::RecordEvent(TEXT("desired_rebuild_breakdown"), FString::Printf(
TEXT("\"frame\":%llu,\"center\":[%d,%d,%d],\"total_ms\":%s,"
"\"core_prepare_ms\":%s,\"core_delta_ms\":%s,\"fallback_ms\":%s,"
"\"overlay_materialize_ms\":%s,\"priority_sort_ms\":%s,"
"\"critical_order_ms\":%s,\"core_keys\":%d,\"candidate_keys\":%d,"
"\"overlay_keys\":%d,\"critical_keys\":%d,\"fallback_reason\":%d,"
"\"changed_lod4plus\":%d,\"materialization_update\":%d,"
"\"priority_order_reused\":%d,\"priority_order_updated\":%d,"
"\"materialization_audit_due\":%d"),
static_cast<unsigned long long>(GFrameCounter), Center.X, Center.Y, Center.Z,
*FString::SanitizeFloat(static_cast<float>(TotalMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(CoreDesiredPreparationMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(CoreDesiredDeltaMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(CoreDesiredFullFallbackMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(OverlayMaterializationMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(PrioritySortMilliseconds)),
*FString::SanitizeFloat(static_cast<float>(CriticalAdmissionOrderMilliseconds)),
CoreDesiredTiles ? CoreDesiredTiles->Num()
: static_cast<int32>(UncachedCoreDesiredTiles.size()),
CandidateKeys.Num(), OverlayDesiredTiles.Num(), CriticalDesiredTiles.Num(),
IncrementalFallbackReason, IncrementalChangedLod4PlusKeyCount,
bIncrementalMaterializationUpdate ? 1 : 0,
bReuseDesiredPriorityOrder ? 1 : 0,
bPriorityOrderUpdated ? 1 : 0,
bShouldAuditDesiredMaterialization ? 1 : 0));
}
} }
void AVoxelWorld::StartPendingCoreRegionCompile() void AVoxelWorld::StartPendingCoreRegionCompile()
@@ -14353,6 +14856,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
"\"core_desired_cache_by_center\":%d," "\"core_desired_cache_by_center\":%d,"
"\"incremental_desired_set\":%d," "\"incremental_desired_set\":%d,"
"\"incremental_desired_audit_every\":%d," "\"incremental_desired_audit_every\":%d,"
"\"incremental_desired_materialization_audit_every\":%d,"
"\"incremental_desired_audit_inject_mismatch\":%d," "\"incremental_desired_audit_inject_mismatch\":%d,"
"\"camera_forward_valid\":%d," "\"camera_forward_valid\":%d,"
"\"near_missing_by_direction\":%s,\"near_submitted_by_direction\":%s," "\"near_missing_by_direction\":%s,\"near_submitted_by_direction\":%s,"
@@ -14391,6 +14895,7 @@ void AVoxelWorld::UpdateChunksAroundPosition(const FVector& CenterPosition, APaw
GVoxelForgeStreamingCoreDesiredCacheByCenter != 0 ? 1 : 0, GVoxelForgeStreamingCoreDesiredCacheByCenter != 0 ? 1 : 0,
GVoxelForgeStreamingIncrementalDesiredSet != 0 ? 1 : 0, GVoxelForgeStreamingIncrementalDesiredSet != 0 ? 1 : 0,
GVoxelForgeStreamingIncrementalDesiredAuditEvery, GVoxelForgeStreamingIncrementalDesiredAuditEvery,
GVoxelForgeStreamingIncrementalDesiredMaterializationAuditEvery,
GVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch, GVoxelForgeStreamingIncrementalDesiredAuditInjectMismatch,
bCameraForwardValid ? 1 : 0, bCameraForwardValid ? 1 : 0,
*IntArray4Json(DesiredMissingByDirection), *IntArray4Json(DesiredMissingByDirection),
+10
View File
@@ -1258,7 +1258,17 @@ public:
TSet<FVoxelTileKey> CachedCoreDesiredTileSet; TSet<FVoxelTileKey> CachedCoreDesiredTileSet;
FIntVector CachedCoreDesiredCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32); FIntVector CachedCoreDesiredCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
bool bHasCachedCoreDesiredTiles = false; bool bHasCachedCoreDesiredTiles = false;
TArray<FVoxelTileKey> CachedDesiredCandidateKeys;
TSet<FVoxelTileKey> CachedDesiredCandidateSet;
TSet<FVoxelTileKey> CachedDesiredBaseVisualTiles;
TSet<FVoxelTileKey> CachedDesiredOverlayTiles;
FIntVector CachedDesiredMaterializationCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
bool bHasCachedDesiredMaterialization = false;
TArray<FVoxelTileKey> CachedDesiredPriorityOrder;
FIntVector CachedDesiredPriorityCenter = FIntVector(MAX_int32, MAX_int32, MAX_int32);
bool bHasCachedDesiredPriorityOrder = false;
int32 IncrementalDesiredCenterCrossings = 0; int32 IncrementalDesiredCenterCrossings = 0;
int32 IncrementalDesiredMaterializationUpdates = 0;
TArray<FVoxelTileKey> DesiredSorted; // critical floor prefix, then true-pawn distance-first TArray<FVoxelTileKey> DesiredSorted; // critical floor prefix, then true-pawn distance-first
TArray<FVoxelTileKey> CriticalDesiredTiles; // occupied/support tile, then heading tile TArray<FVoxelTileKey> CriticalDesiredTiles; // occupied/support tile, then heading tile
// Ancestor -> desired descendants, rebuilt with DesiredSorted. Reconciliation uses this // Ancestor -> desired descendants, rebuilt with DesiredSorted. Reconciliation uses this
@@ -0,0 +1,58 @@
# L166 — Incremental streaming update
**Result:** Core and overlay candidate materialization now apply entering/leaving deltas, and same-center priority order updates merge only changed keys. Seven 30 m/s W8 pairs show lower median rebuild p95, tick p99, worst tick, near readiness, forward-cone readiness, and far holes. The strict pairwise bar is not fully clean: one worst-tick pair, three far-hole pairs, and four arrival pairs regressed slightly. Set/order audits and the fast tier pass.
## Count first
Stage costs below are milliseconds per desired rebuild, reported as the median of the seven runs' p50 and p95 values. Each run's window is its 12-second move at 30 m/s; there were 44–45 center crossings per run.
| Stage | `assembler/world` p50 / p95 | Incremental p50 / p95 |
| --- | ---: | ---: |
| Core query/preparation | 0.832 / 2.229 | 0.0002 / 0.142 |
| Core crossing delta | — | 0.000 / 0.138 |
| UE overlay materialization | 0.350 / 0.723 | 0.103 / 0.356 |
| Priority sort/order | 0.306 / 0.656 | 0.038 / 0.437 |
| Critical admission order | 0.063 / 0.134 | 0.065 / 0.222 |
The incremental Core path updated on 98.4% of desired rebuilds. Priority order merged key deltas on 52.2%, reused the cached order unchanged on 33.5%, and fully sorted on the remaining 14.3% (the center-crossing updates). LOD4+ crossings caused five full Core-query fallbacks per 12-second move; their median per-run maximum was 1.19 ms and their median total was 5.25 ms spread across those five events.
The median worst tick was 11.652 ms versus 12.980 ms for the baseline. The largest candidate worst-tick sample was 12.764 ms at a center crossing: its priority sort took 1.311 ms, overlay materialization 0.915 ms, and Core delta 0.601 ms; no LOD4+ fallback occurred on that tick. Priority sorting remained the largest measured substage in that worst tick. Fallbacks were not the source of the remaining worst ticks.
## Change and equality checks
`BuildCoreDesiredTiles` now retains the Core visual set, UE overlay set, candidate set, and priority order. It applies Core and overlay membership deltas to the persistent sets. Live anchor, gate, and support inputs are still walked into a temporary overlay union on each trigger. At an unchanged center, it removes leaving keys and merges sorted entering keys into the cached priority order. Center changes still require a full priority sort because the distance comparator changes. The critical-key promotion list is still rebuilt on each trigger.
The equality audit independently obtains a full Core desired set, rebuilds the overlay reference, and compares candidate and collision-only sets. A second audit builds the full prioritized and critical order and compares it with the incremental result. Across the seven measured candidate runs, all 28 sampled set audits and all 28 order audits passed. A positive materialization-control run passed 13 additional set and order audits. The injected mismatch control logged `missing=1` and failed with a nonzero UE exit (`-11`), as expected.
## Seven interleaved pairs
Each pair compared `assembler/world` at `9c70053` against this worktree, with a 12-second 30 m/s move, 5-second dwell, and eight effective tile slots. Values are medians across seven runs; paired gates are stated separately.
| Metric | `assembler/world` | Incremental | Paired result |
| --- | ---: | ---: | --- |
| `BuildDesiredTiles` p95 | 7.281 ms | 3.738 ms | 48.7% lower median |
| Game-thread tick p99 | 10.521 ms | 7.420 ms | Lower in all 7 pairs; candidate range 7.031–7.526 ms |
| Worst tick | 12.980 ms | 11.652 ms | Lower in 6/7 pairs; pair 5 was 0.347 ms higher |
| Near LOD0–2 readiness | 41.995% | 43.668% | Better in all 7 pairs |
| 55° forward-cone readiness | 14.779% | 16.116% | Better in all 7 pairs |
| Far holes while moving | 3.197% | 2.739% | Lower median; better in 4/7 pairs |
| First settled LOD0–2 arrival | 2.432 s | 2.439 s | Paired median delta +0.002 s; later in 4/7 pairs |
The paired far-hole deltas were −0.072, +0.144, −0.059, +0.301, −1.052, −0.773, and +0.069 percentage points. The L165 +0.291 pp result is within this observed spread; L166 also had one +0.301 pp pair, but the seven-pair median delta was −0.059 pp and four pairs improved. This is consistent with substantial run-to-run variation, not a consistent far-hole regression; it does not establish a strict no-regression guarantee.
Null-RHI produced no camera-forward samples, so the 55° cone uses the travel-direction fallback. The arrival measure also varied sharply between pairs, including two candidate runs that settled 0.22–0.26 s earlier. Its paired median was effectively flat, but the strict per-pair arrival gate did not pass.
## Checks and limits
- VFHost Linux Development build succeeded with `-NoUBA -MaxParallelActions=4`; `VFHost/PluginDeps/VoxelForge/Source` was restored to `/home/jahni/VoxelM/Plugins/VoxelForge/Source`.
- `verify.py fast` passed: 268 checks, 0 failures, 5 diagnostics, 281.5 s. Clang/GCC W1/W8/W12 determinism and canonical geometry checks passed.
- The live movement `TileHashDump` hook was enabled but wrote zero records for both conditions, so that direct runtime geometry comparison is inconclusive. No meshing or geometry-generation code changed; the fast-tier geometry hashes and sampled full-rebuild order audit passed.
- `git diff --check` and runner `py_compile` passed. W12 movement timing, camera-forward cone capture, and separate movement runs on each of the four owner layers were not run.
- The strict movement summary fails `paired_worst_tick_no_worse`, `paired_far_holes_no_worse`, and `paired_arrival_no_worse`; p99, near readiness, forward-cone readiness, audits, and median rebuild-p95 gates pass.
Changed Unreal `Source/` files:
- `Source/VoxelForge/Private/VoxelWorld.cpp`
- `Source/VoxelForge/Public/VoxelWorld.h`
Logs, traces, summary, and verifier artifacts are under `Saved/Linux/l166/`.
@@ -0,0 +1,685 @@
#!/usr/bin/env python3
"""Interleaved L166 UE movement runs for incremental desired-set rebuilds."""
from __future__ import annotations
import argparse
import json
import math
import statistics
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[4]
HOST = Path("/home/jahni/VFHost")
EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd")
OUT = ROOT / "Saved/Linux/l166/ue/movement"
HOST_SAVED = HOST / "Saved/Linux/l166"
HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l166"
SPAWN_M = (19.166015625, -14.2001953125, -753.3740234375)
MOVE_START_S = 6.0
CONDITIONS = {
"assembler_world": {
"wait_for_critical": 1, "apply_budget_ms": 0.0,
"forward_first": 0, "rebuild_on_region_completion": 1,
"core_desired_cache": 0, "incremental_desired": 0,
"audit_every": 0, "materialization_audit_every": 0, "inject_mismatch": 0,
},
"incremental": {
"wait_for_critical": 1, "apply_budget_ms": 0.0,
"forward_first": 0, "rebuild_on_region_completion": 1,
"core_desired_cache": 0, "incremental_desired": 1,
"audit_every": 10, "materialization_audit_every": 0, "inject_mismatch": 0,
},
}
ORDER = {
1: ("assembler_world", "incremental"),
2: ("incremental", "assembler_world"),
3: ("assembler_world", "incremental"),
4: ("incremental", "assembler_world"),
5: ("assembler_world", "incremental"),
6: ("incremental", "assembler_world"),
7: ("assembler_world", "incremental"),
}
def percentile(values: list[float], fraction: float = 0.99) -> float | None:
if not values:
return None
ordered = sorted(values)
return ordered[max(0, min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1))]
def read_events(path: Path) -> list[dict]:
rows = []
if path.is_file():
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
if not line.startswith("{"):
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
continue
return rows
def mean(values: list[float]) -> float | None:
return statistics.mean(values) if values else None
def desired_breakdown(events: list[dict], move_start_s: float, stop_s: float) -> dict:
rows = [event for event in events
if event.get("type") == "desired_rebuild_breakdown"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s]
keys = ("total_ms", "core_prepare_ms", "core_delta_ms", "fallback_ms",
"overlay_materialize_ms", "priority_sort_ms", "critical_order_ms")
result = {"rebuild_count": len(rows)}
result["incremental_materialization_update_count"] = sum(
int(event.get("materialization_update", 0)) != 0 for event in rows)
result["priority_order_reuse_count"] = sum(
int(event.get("priority_order_reused", 0)) != 0 for event in rows)
result["priority_order_update_count"] = sum(
int(event.get("priority_order_updated", 0)) != 0 for event in rows)
result["incremental_materialization_update_share_pct"] = percent(
result["incremental_materialization_update_count"], len(rows))
result["priority_order_reuse_share_pct"] = percent(
result["priority_order_reuse_count"], len(rows))
result["priority_order_update_share_pct"] = percent(
result["priority_order_update_count"], len(rows))
for key in keys:
values = [float(event.get(key, 0.0)) for event in rows]
result[f"{key}_p50"] = statistics.median(values) if values else None
result[f"{key}_p95"] = percentile(values, 0.95)
result[f"{key}_max"] = max(values) if values else None
fallback_rows = [event for event in events
if event.get("type") == "core_desired_cache"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s
and int(event.get("full_fallback", 0)) != 0]
result["fallback_count"] = len(fallback_rows)
result["lod4plus_fallback_count"] = sum(
int(event.get("fallback_reason", 0)) == 1 for event in fallback_rows)
result["other_fallback_count"] = sum(
int(event.get("fallback_reason", 0)) == 2 for event in fallback_rows)
result["fallback_cost_sum_ms"] = sum(
float(event.get("fallback_ms", 0.0)) for event in fallback_rows)
result["fallback_cost_max_ms"] = max(
(float(event.get("fallback_ms", 0.0)) for event in fallback_rows), default=None)
return result
def percent(numerator: int | float, denominator: int | float) -> float | None:
return 100.0 * numerator / denominator if denominator > 0 else None
def run(condition: str, repetition: int, tile_hash_dump: Path | None = None,
expected_mismatch: bool = False) -> dict:
config = CONDITIONS[condition]
run_name = f"fast-r{repetition}"
out = OUT / condition
host_out = HOST_PLUGIN_SAVED / condition / "output"
userdir = HOST_SAVED / "user" / condition / run_name
zendata = HOST_SAVED / "zen" / condition / run_name
ddc = HOST_SAVED / "DDC"
trace = out / f"{run_name}.trace.jsonl"
abslog = out / f"{run_name}.log"
stdout = out / f"{run_name}.stdout.log"
export = host_out / run_name
for directory in (out, export, userdir, zendata, ddc):
directory.mkdir(parents=True, exist_ok=True)
trace.unlink(missing_ok=True)
spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWN_M)
command = [
str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended",
"-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch",
"-notraceserver", "-nocrashreports", "-DisablePlugins=AndroidFileServer",
"-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}",
f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0",
"-archetype=TunnelNetwork", "-slot=0", "-modes=export", "-gameconfig=1",
"-coreworldtest=1", "-ownerfourstrateplan=1", "-corevoxelsizecm=25",
"-coreislandtest=0", "-corefallbacktest=0", "-coreofftest=0",
"-coreplantest=0", "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings",
"-corehybridmesher=0", f"-out={export}", "-voxel.StreamingCoarseFirst=1",
"-voxel.StreamingDynamicNearReserve=1",
"-voxel.StreamingLegacyPendingResultCap=0",
"-voxel.CoreRegionConcurrentAdmission=1",
"-voxel.StreamingResultBacklogCap=256", "-voxel.StreamingReadyQueueDepth=8",
"-voxel.StreamingBackgroundWorkerCap=1", "-voxel.CoreColumnCacheDirectMapped=0",
"-voxel.StreamingFarApplySupplement=1",
f"-voxel.StreamingFastTravelWaitForAllCriticalTiles={config['wait_for_critical']}",
f"-voxel.StreamingFastTravelApplyBudgetMilliseconds={config['apply_budget_ms']:.1f}",
f"-voxel.StreamingFastTravelForwardFirst={config['forward_first']}",
f"-voxel.StreamingRebuildOnCoreRegionCompletion={config['rebuild_on_region_completion']}",
f"-voxel.StreamingCoreDesiredCacheByCenter={config['core_desired_cache']}",
f"-voxel.StreamingIncrementalDesiredSet={config['incremental_desired']}",
f"-voxel.StreamingIncrementalDesiredAuditEvery={config['audit_every']}",
f"-voxel.StreamingIncrementalDesiredMaterializationAuditEvery={config['materialization_audit_every']}",
f"-voxel.StreamingIncrementalDesiredAuditInjectMismatch={config['inject_mismatch']}",
"-voxel.CoreMCGradientMemoization=1", "-voxel.StreamingCollisionLookahead=1",
"-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=3000.0",
f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}",
"-voxel.TestMoveDurationSeconds=12.0", "-voxel.TestExitSeconds=0.0",
"-voxel.TestMoveDirection=1,0,0", "-voxel.StreamingSchedulerPolicy=3",
"-voxel.TestExitOnSteady=0", "-coreworldstreamtrace=1",
f"-voxel.StartupTraceSpawnCm={spawn_cm}",
"-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1",
"-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}",
f"-abslog={abslog}",
]
if tile_hash_dump is not None:
tile_hash_dump.parent.mkdir(parents=True, exist_ok=True)
tile_hash_dump.unlink(missing_ok=True)
command.append(f"-voxel.TileHashDump={tile_hash_dump.resolve()}")
wrapped = ["taskset", "-c", "0-11", "timeout", "1740s", *command]
started = time.monotonic()
with stdout.open("w", encoding="utf-8") as stream:
process = subprocess.run(
wrapped, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
check=False, timeout=1740)
events = read_events(trace)
if expected_mismatch:
diagnostic = "VoxelForgeIncrementalDesiredAudit"
logs = "\n".join(path.read_text(encoding="utf-8", errors="replace")
for path in (abslog, stdout) if path.is_file())
audit_events = [event for event in events
if event.get("type") in {"incremental_desired_audit",
"incremental_desired_order_audit"}]
failed_audit = any(int(event.get("pass", 1)) == 0 for event in audit_events)
failed_audit = failed_audit or (
diagnostic in logs and "mismatch" in logs.lower())
if process.returncode == 0 or diagnostic not in logs or "mismatch" not in logs.lower():
raise RuntimeError(
f"injected mismatch did not fail as expected: exit={process.returncode}; "
f"audit_events={len(audit_events)}; log={abslog}")
print("[L166MismatchControl] " + json.dumps({
"exit_code": process.returncode,
"diagnostic_seen": True,
"failed_audit_event_seen": failed_audit,
"trace": str(trace.relative_to(ROOT)),
"log": str(abslog.relative_to(ROOT)),
}, sort_keys=True), flush=True)
return {"condition": condition, "exit_code": process.returncode,
"failed_audit_event_seen": failed_audit}
samples = [event for event in events if event.get("type") == "streaming_sample"]
moving = [event for event in samples if float(event.get("speed_cm_s", 0.0)) >= 50.0]
stop_event = next((event for event in events
if event.get("type") == "streaming_test_move_stopped"), None)
backend = next((event for event in events if event.get("type") == "worker_backend_config"), {})
scheduler_config = next((event for event in events
if event.get("type") == "scheduler_admission"), {})
worker_cap = next((event for event in events
if event.get("type") == "streaming_worker_cap_readback"), {})
if stop_event is None:
raise RuntimeError(f"missing move-stop event: {trace}")
stop_s = float(stop_event.get("t_s", 0.0))
move_start_s = min((float(event.get("t_s", 0.0)) for event in moving), default=float("inf"))
dwell = [event for event in samples if float(event.get("t_s", 0.0)) >= stop_s
and float(event.get("speed_cm_s", 0.0)) < 50.0]
arrival = [event for event in samples if float(event.get("t_s", 0.0)) < move_start_s
and float(event.get("speed_cm_s", 0.0)) < 50.0]
desired_stage_breakdown = desired_breakdown(events, move_start_s, stop_s)
if process.returncode or len(moving) < 40 or len(dwell) < 16:
raise RuntimeError(f"incomplete {condition} run: {trace}; exit={process.returncode}; "
f"move={len(moving)} dwell={len(dwell)}")
if (int(backend.get("streaming_fast_travel_wait_for_all_critical_tiles", -1))
!= config["wait_for_critical"]
or abs(float(backend.get("streaming_fast_travel_apply_budget_ms", -1.0))
- config["apply_budget_ms"]) > 0.01
or int(worker_cap.get("streaming_background_worker_cap", -1)) != 1
or int(backend.get("effective_tile_slots", -1)) != 8
or int(backend.get("streaming_far_apply_supplement", -1)) != 1
or int(scheduler_config.get("forward_first", -1)) != config["forward_first"]
or int(scheduler_config.get("rebuild_on_core_region_completion", -1))
!= config["rebuild_on_region_completion"]
or int(scheduler_config.get("core_desired_cache_by_center", -1))
!= config["core_desired_cache"]
or int(scheduler_config.get("incremental_desired_set", -1))
!= config["incremental_desired"]
or int(scheduler_config.get("incremental_desired_audit_every", -1))
!= config["audit_every"]
or int(scheduler_config.get("incremental_desired_materialization_audit_every", -1))
!= config["materialization_audit_every"]
or int(scheduler_config.get("incremental_desired_audit_inject_mismatch", -1))
!= config["inject_mismatch"]):
raise RuntimeError(f"CVar/worker readback mismatch: {backend}; {worker_cap}; {scheduler_config}")
moving_metrics = []
for event in moving:
desired = event.get("desired_by_lod", [0] * 9)
resident = event.get("resident_by_lod", [0] * 9)
near_desired = sum(desired[:3])
cone_desired = int(event.get("near_forward_cone_desired", 0))
view_cone_desired = int(event.get("near_view_forward_cone_desired", 0))
view_valid = bool(event.get("near_view_forward_cone_valid", 0))
weighted_desired = float(event.get("near_forward_weighted_desired", 0.0))
travel_cone_ready = percent(
int(event.get("near_forward_cone_resident", 0)), cone_desired)
view_cone_ready = percent(
int(event.get("near_view_forward_cone_resident", 0)), view_cone_desired)
moving_metrics.append({
"near_ready_pct": percent(sum(resident[:3]), near_desired) or 0.0,
"lod0_ready_pct": percent(resident[0], desired[0]) or 0.0,
"lod1_ready_pct": percent(resident[1], desired[1]) or 0.0,
"lod2_ready_pct": percent(resident[2], desired[2]) or 0.0,
"far_hole_pct": percent(
sum(desired[3:]) - sum(event.get("covered_by_lod", resident)[3:]),
sum(desired[3:])) or 0.0,
"ahead_share_pct": percent(int(event.get("near_ahead_desired", 0)), near_desired),
"behind_share_pct": percent(int(event.get("near_behind_desired", 0)), near_desired),
"side_share_pct": 100.0 - (
percent(int(event.get("near_ahead_desired", 0)), near_desired) or 0.0
) - (
percent(int(event.get("near_behind_desired", 0)), near_desired) or 0.0
),
"ahead_ready_pct": percent(
int(event.get("near_ahead_resident", 0)),
int(event.get("near_ahead_desired", 0))),
"behind_ready_pct": percent(
int(event.get("near_behind_resident", 0)),
int(event.get("near_behind_desired", 0))),
"travel_forward_cone_ready_pct": travel_cone_ready,
"view_forward_cone_ready_pct": view_cone_ready,
"forward_cone_ready_pct": view_cone_ready if view_valid else travel_cone_ready,
"forward_cone_source": "camera" if view_valid else "travel_fallback",
"forward_cone_share_pct": percent(cone_desired, near_desired),
"forward_weighted_ready_pct": percent(
float(event.get("near_forward_weighted_resident", 0.0)), weighted_desired),
})
arrival_near_settled = next((event for event in arrival
if sum(event.get("desired_by_lod", [0] * 9)[:3]) > 0
and sum(event.get("settled_by_lod", [0] * 9)[:3])
>= sum(event.get("desired_by_lod", [0] * 9)[:3])), None)
dwell_complete = next((event for event in dwell
if sum(event.get("resident_by_lod", [0] * 9)[:3])
>= sum(event.get("desired_by_lod", [0] * 9)[:3])
and sum(event.get("desired_by_lod", [0] * 9)[3:])
<= sum(event.get("covered_by_lod", [0] * 9)[3:])), None)
work = [event for event in events if event.get("type") == "game_thread_work_frame"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s]
applies = {event.get("frame"): event for event in events
if event.get("type") == "apply_frame"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s}
scheduler = {event.get("frame"): event for event in events
if event.get("type") == "scheduler_admission"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s}
frames = []
for event in work:
apply = applies.get(event.get("frame"), {})
admit = scheduler.get(event.get("frame"), {})
frames.append({
"frame": event.get("frame"),
"tick_ms": float(event.get("tick_ms", 0.0)),
"update_chunks_ms": float(event.get("update_chunks_ms", 0.0)),
"pending_drain_ms": float(event.get("pending_drain_ms", 0.0)),
"apply_ms": float(apply.get("apply_ms", 0.0)),
"apply_budget_ms": float(apply.get("apply_budget_ms", 0.0)),
"meshes_applied": int(apply.get("meshes_applied", 0)),
"time_budget_hit": int(apply.get("time_limit_reached", 0)),
"desired_rebuild_ms": float(admit.get("desired_rebuild_ms", 0.0)),
"desired_rebuilt": int(admit.get("desired_rebuilt", 0)),
"submit_tiles_ms": float(admit.get("submit_tiles_ms", 0.0)),
"level_admit_ms": float(admit.get("level_admit_ms", 0.0)),
})
p99_field = lambda key: percentile([float(row[key]) for row in frames])
max_field = lambda key: max((float(row[key]) for row in frames), default=None)
rebuild_frames = [row for row in frames if row["desired_rebuilt"]]
rebuild_costs = [float(row["desired_rebuild_ms"]) for row in rebuild_frames]
moving_scheduler = [event for event in scheduler.values()]
center_crossings = 0
previous_center = None
for event in sorted(moving_scheduler, key=lambda item: int(item.get("frame", 0))):
center = tuple(event.get("center", ()))
if previous_center is not None and center != previous_center:
center_crossings += 1
previous_center = center
core_requests = [event for event in events
if event.get("type") == "core_region_compile_request"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s]
same_center_core_requests = sum(
event.get("reason") == "desired_set_changed_same_center" for event in core_requests)
core_desired_queries = [event for event in events
if event.get("type") == "core_desired_cache"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s]
core_desired_query_ms = [float(event.get("query_ms", 0.0))
for event in core_desired_queries if float(event.get("query_ms", 0.0)) > 0.0]
core_desired_cache_hits = sum(int(event.get("hit", 0)) for event in core_desired_queries)
incremental_audits = [event for event in events
if event.get("type") == "incremental_desired_audit"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s]
incremental_audit_failures = sum(int(event.get("pass", 1)) == 0
for event in incremental_audits)
order_audits = [event for event in events
if event.get("type") == "incremental_desired_order_audit"
and move_start_s <= float(event.get("t_s", 0.0)) < stop_s]
order_audit_failures = sum(int(event.get("pass", 1)) == 0 for event in order_audits)
materialization_audits = [event for event in incremental_audits
if int(event.get("materialization_audit", 0)) != 0]
if config["incremental_desired"] and (
not incremental_audits or incremental_audit_failures
or not order_audits or order_audit_failures
or (config["materialization_audit_every"] > 0
and not materialization_audits)):
raise RuntimeError(
f"incremental desired equality audit missing/failed: {trace}; "
f"set={len(incremental_audits)}/{incremental_audit_failures} "
f"order={len(order_audits)}/{order_audit_failures} "
f"materialization={len(materialization_audits)}")
direction_submissions = [event.get("near_submitted_by_direction", [0, 0, 0, 0])
for event in moving_scheduler]
direction_submissions = [counts for counts in direction_submissions
if isinstance(counts, list) and len(counts) == 4]
direction_submission_totals = [
sum(int(counts[index]) for counts in direction_submissions) for index in range(4)
]
first_admission_order = next((event.get("near_admission_order_first", [])
for event in moving_scheduler
if event.get("near_admission_order_first")), [])
camera_samples = sum(bool(event.get("near_view_forward_cone_valid", 0)) for event in moving)
dwell_complete_s = (float(dwell_complete.get("t_s", 0.0)) - stop_s
if dwell_complete else None)
row = {
"condition": condition,
"repetition": repetition,
"speed_mps": 30.0,
"wait_for_all_critical_tiles": config["wait_for_critical"],
"fast_travel_apply_budget_ms": config["apply_budget_ms"],
"forward_first": config["forward_first"],
"rebuild_on_core_region_completion": config["rebuild_on_region_completion"],
"core_desired_cache_by_center": config["core_desired_cache"],
"incremental_desired_set": config["incremental_desired"],
"incremental_desired_audit_every": config["audit_every"],
"incremental_desired_materialization_audit_every": config["materialization_audit_every"],
"incremental_desired_audit_count": len(incremental_audits),
"incremental_desired_audit_failures": incremental_audit_failures,
"incremental_desired_order_audit_count": len(order_audits),
"incremental_desired_order_audit_failures": order_audit_failures,
"incremental_desired_materialization_audit_count": len(materialization_audits),
"effective_tile_slots": backend.get("effective_tile_slots"),
"exit_code": process.returncode,
"wall_s": round(time.monotonic() - started, 3),
"moving_samples": len(moving),
"dwell_samples": len(dwell),
"arrival_samples": len(arrival),
"near_ready_moving_mean_pct": mean([m["near_ready_pct"] for m in moving_metrics]),
"lod0_ready_moving_mean_pct": mean([m["lod0_ready_pct"] for m in moving_metrics]),
"lod1_ready_moving_mean_pct": mean([m["lod1_ready_pct"] for m in moving_metrics]),
"lod2_ready_moving_mean_pct": mean([m["lod2_ready_pct"] for m in moving_metrics]),
"far_holes_moving_mean_pct": mean([m["far_hole_pct"] for m in moving_metrics]),
"near_ahead_desired_share_pct": mean([m["ahead_share_pct"] for m in moving_metrics
if m["ahead_share_pct"] is not None]),
"near_behind_desired_share_pct": mean([m["behind_share_pct"] for m in moving_metrics
if m["behind_share_pct"] is not None]),
"near_side_desired_share_pct": mean([m["side_share_pct"] for m in moving_metrics]),
"near_ahead_ready_pct": mean([m["ahead_ready_pct"] for m in moving_metrics
if m["ahead_ready_pct"] is not None]),
"near_behind_ready_pct": mean([m["behind_ready_pct"] for m in moving_metrics
if m["behind_ready_pct"] is not None]),
"near_forward_cone_half_angle_deg": 55,
"near_forward_cone_desired_share_pct": mean([m["forward_cone_share_pct"]
for m in moving_metrics
if m["forward_cone_share_pct"] is not None]),
"near_forward_cone_ready_pct": mean([m["forward_cone_ready_pct"]
for m in moving_metrics
if m["forward_cone_ready_pct"] is not None]),
"near_travel_forward_cone_ready_pct": mean(
[m["travel_forward_cone_ready_pct"] for m in moving_metrics
if m["travel_forward_cone_ready_pct"] is not None]),
"near_view_forward_cone_ready_pct": mean(
[m["view_forward_cone_ready_pct"] for m in moving_metrics
if m["view_forward_cone_ready_pct"] is not None]),
"camera_forward_samples": camera_samples,
"camera_forward_sample_share_pct": percent(camera_samples, len(moving)),
"near_forward_cosine_weighted_ready_pct": mean(
[m["forward_weighted_ready_pct"] for m in moving_metrics
if m["forward_weighted_ready_pct"] is not None]),
"arrival_lod02_settled_t_s": (
float(arrival_near_settled.get("t_s", 0.0)) if arrival_near_settled else None),
"near_ready_dwell_end_pct": (100.0 * sum(dwell[-1].get("resident_by_lod", [0] * 9)[:3])
/ max(1, sum(dwell[-1].get("desired_by_lod", [0] * 9)[:3]))),
"far_holes_dwell_end_pct": (100.0 * (
sum(dwell[-1].get("desired_by_lod", [0] * 9)[3:])
- sum(dwell[-1].get("covered_by_lod", [0] * 9)[3:]))
/ max(1, sum(dwell[-1].get("desired_by_lod", [0] * 9)[3:]))),
"dwell_complete_after_stop_s": dwell_complete_s,
"game_thread_tick_p99_ms": p99_field("tick_ms"),
"game_thread_tick_max_ms": max_field("tick_ms"),
"stream_update_p99_ms": p99_field("update_chunks_ms"),
"pending_drain_p99_ms": p99_field("pending_drain_ms"),
"apply_p99_ms": p99_field("apply_ms"),
"apply_max_ms": max_field("apply_ms"),
"apply_budget_min_ms": min((f["apply_budget_ms"] for f in frames), default=None),
"apply_budget_max_ms": max((f["apply_budget_ms"] for f in frames), default=None),
"apply_time_budget_hit_frames": sum(f["time_budget_hit"] for f in frames),
"apply_frames": sum(f["meshes_applied"] > 0 for f in frames),
"apply_meshes_total": sum(f["meshes_applied"] for f in frames),
"desired_rebuild_p99_ms": p99_field("desired_rebuild_ms"),
"desired_rebuild_count": len(rebuild_frames),
"desired_rebuilds_per_second": len(rebuild_frames) / 12.0,
"center_crossings": center_crossings,
"same_center_core_region_requests": same_center_core_requests,
"core_desired_cache_hit_count": core_desired_cache_hits,
"core_desired_cache_hit_share_pct": percent(
core_desired_cache_hits, len(core_desired_queries)),
"core_desired_query_nonzero_p50_ms": statistics.median(core_desired_query_ms)
if core_desired_query_ms else None,
"core_desired_query_nonzero_p95_ms": percentile(core_desired_query_ms, 0.95),
"core_desired_query_nonzero_max_ms": max(core_desired_query_ms)
if core_desired_query_ms else None,
"desired_rebuild_nonzero_p50_ms": statistics.median(rebuild_costs)
if rebuild_costs else None,
"desired_rebuild_nonzero_p95_ms": percentile(rebuild_costs, 0.95),
"desired_rebuild_nonzero_max_ms": max(rebuild_costs) if rebuild_costs else None,
"desired_breakdown": desired_stage_breakdown,
"near_submitted_by_direction": direction_submission_totals,
"near_admission_first12": first_admission_order,
"submit_tiles_p99_ms": p99_field("submit_tiles_ms"),
"level_admit_p99_ms": p99_field("level_admit_ms"),
"top_tick_frames": sorted(frames, key=lambda f: f["tick_ms"], reverse=True)[:5],
"trace": str(trace.relative_to(ROOT)),
"log": str(abslog.relative_to(ROOT)),
}
print("[L166Movement] " + json.dumps(row, sort_keys=True), flush=True)
return row
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--smoke", action="store_true",
help="run one incremental trace to validate readback and equality audits")
parser.add_argument("--count-only", action="store_true",
help="run one baseline/candidate pair for desired-stage cost attribution")
parser.add_argument("--geometry-hash-check", action="store_true",
help="compare UE per-tile geometry hashes for baseline and incremental desired sets")
parser.add_argument("--controls-only", action="store_true",
help="run a per-crossing equality audit and injected mismatch negative control")
parser.add_argument("--summarize-existing", action="store_true",
help="summarize the fourteen completed rows in runs-partial.json without rerunning UE")
args = parser.parse_args()
missing = [path for path in (EDITOR, HOST / "VFHost.uproject") if not path.exists()]
if missing:
raise SystemExit(f"missing UE runner input: {missing}")
OUT.mkdir(parents=True, exist_ok=True)
if args.geometry_hash_check:
hash_dir = OUT / "geometry-hash"
baseline_path = hash_dir / "assembler-world.json"
candidate_path = hash_dir / "incremental.json"
run("assembler_world", 99, baseline_path)
run("incremental", 99, candidate_path)
baseline_data = json.loads(baseline_path.read_text(encoding="utf-8"))
candidate_data = json.loads(candidate_path.read_text(encoding="utf-8"))
key_fields = ("level", "tile_x", "tile_y", "tile_z")
def indexed(data: dict) -> dict[tuple[int, ...], tuple]:
result = {}
for record in data.get("records", []):
key = tuple(int(record[field]) for field in key_fields)
result[key] = (str(record.get("geometry_hash", "")),
int(record.get("triangle_count", -1)),
bool(record.get("empty", False)))
return result
baseline_hashes = indexed(baseline_data)
candidate_hashes = indexed(candidate_data)
common = baseline_hashes.keys() & candidate_hashes.keys()
mismatches = [key for key in common
if baseline_hashes[key] != candidate_hashes[key]]
result = {
"baseline_records": len(baseline_hashes),
"candidate_records": len(candidate_hashes),
"common_records": len(common),
"hash_mismatches": len(mismatches),
"baseline_only_records": len(baseline_hashes.keys() - candidate_hashes.keys()),
"candidate_only_records": len(candidate_hashes.keys() - baseline_hashes.keys()),
"baseline_dump": str(baseline_path.relative_to(ROOT)),
"candidate_dump": str(candidate_path.relative_to(ROOT)),
}
(hash_dir / "comparison.json").write_text(
json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print("[L166GeometryHash] " + json.dumps(result, sort_keys=True), flush=True)
return 0 if len(common) >= 100 and not mismatches else 1
if args.count_only:
run("assembler_world", 99)
run("incremental", 99)
return 0
if args.smoke:
run("incremental", 0)
return 0
if args.controls_only:
control = dict(CONDITIONS["incremental"])
control["audit_every"] = 0
control["materialization_audit_every"] = 25
control["inject_mismatch"] = 0
CONDITIONS["materialization_audit_control"] = control
run("materialization_audit_control", 0)
mismatch = dict(control)
mismatch["materialization_audit_every"] = 1
mismatch["inject_mismatch"] = 1
CONDITIONS["injected_materialization_mismatch_control"] = mismatch
run("injected_materialization_mismatch_control", 0, expected_mismatch=True)
return 0
partial = OUT / "runs-partial.json"
if args.summarize_existing:
if not partial.is_file():
raise RuntimeError(f"no saved movement rows: {partial}")
rows = json.loads(partial.read_text(encoding="utf-8"))
if len(rows) != 14:
raise RuntimeError(f"expected fourteen completed movement rows, found {len(rows)}")
else:
rows = []
# Both conditions use this editor binary. Warm each path before measured pairs.
run("assembler_world", 0)
run("incremental", 0)
for repetition in range(1, 8):
for condition in ORDER[repetition]:
row = run(condition, repetition)
rows.append(row)
partial.write_text(json.dumps(rows, indent=2, sort_keys=True) + "\n",
encoding="utf-8")
metrics = (
"near_ready_moving_mean_pct", "far_holes_moving_mean_pct",
"game_thread_tick_p99_ms", "game_thread_tick_max_ms",
"arrival_lod02_settled_t_s", "dwell_complete_after_stop_s",
"near_ahead_desired_share_pct", "near_behind_desired_share_pct",
"near_side_desired_share_pct", "near_forward_cone_desired_share_pct",
"near_ahead_ready_pct", "near_behind_ready_pct",
"near_forward_cone_ready_pct", "near_forward_cosine_weighted_ready_pct",
"near_travel_forward_cone_ready_pct",
"apply_p99_ms", "desired_rebuild_p99_ms", "desired_rebuild_count",
"desired_rebuild_nonzero_p95_ms", "center_crossings",
"same_center_core_region_requests", "level_admit_p99_ms",
"core_desired_cache_hit_count", "core_desired_query_nonzero_p95_ms",
)
medians = {}
for condition in CONDITIONS:
medians[condition] = {}
for metric in metrics:
values = [row[metric] for row in rows
if row["condition"] == condition and row[metric] is not None]
medians[condition][metric] = statistics.median(values) if values else None
pairs = []
for repetition in range(1, 8):
by_condition = {row["condition"]: row for row in rows
if row["repetition"] == repetition}
candidate = "incremental"
pairs.append({
"repetition": repetition,
"candidate": candidate,
**{f"{metric}_delta": by_condition[candidate][metric]
- by_condition["assembler_world"][metric]
for metric in metrics if by_condition[candidate][metric] is not None
and by_condition["assembler_world"][metric] is not None},
})
by_pair = {
repetition: {row["condition"]: row for row in rows if row["repetition"] == repetition}
for repetition in range(1, 8)
}
candidate_rows = [row for row in rows if row["condition"] == "incremental"]
baseline_rows = [row for row in rows if row["condition"] == "assembler_world"]
candidate_gates = {
"every_candidate_run_p99_le_11_ms": all(
row["game_thread_tick_p99_ms"] <= 11.0 for row in candidate_rows),
"paired_p99_no_worse": all(
by_pair[rep]["incremental"]["game_thread_tick_p99_ms"]
<= by_pair[rep]["assembler_world"]["game_thread_tick_p99_ms"]
for rep in by_pair),
"paired_worst_tick_no_worse": all(
by_pair[rep]["incremental"]["game_thread_tick_max_ms"]
<= by_pair[rep]["assembler_world"]["game_thread_tick_max_ms"]
for rep in by_pair),
"paired_near_all_around_no_worse": all(
by_pair[rep]["incremental"]["near_ready_moving_mean_pct"]
>= by_pair[rep]["assembler_world"]["near_ready_moving_mean_pct"]
for rep in by_pair),
"paired_forward_cone_no_worse": all(
by_pair[rep]["incremental"]["near_forward_cone_ready_pct"]
>= by_pair[rep]["assembler_world"]["near_forward_cone_ready_pct"]
for rep in by_pair),
"paired_far_holes_no_worse": all(
by_pair[rep]["incremental"]["far_holes_moving_mean_pct"]
<= by_pair[rep]["assembler_world"]["far_holes_moving_mean_pct"]
for rep in by_pair),
"paired_arrival_no_worse": all(
by_pair[rep]["incremental"]["arrival_lod02_settled_t_s"] is not None
and by_pair[rep]["assembler_world"]["arrival_lod02_settled_t_s"] is not None
and by_pair[rep]["incremental"]["arrival_lod02_settled_t_s"]
<= by_pair[rep]["assembler_world"]["arrival_lod02_settled_t_s"]
for rep in by_pair),
"all_candidate_equality_audits_pass": all(
row["incremental_desired_audit_count"] > 0
and row["incremental_desired_audit_failures"] == 0
and row["incremental_desired_order_audit_count"] > 0
and row["incremental_desired_order_audit_failures"] == 0
for row in candidate_rows),
"median_build_desired_p95_down": statistics.median(
row["desired_rebuild_nonzero_p95_ms"] for row in candidate_rows)
< statistics.median(row["desired_rebuild_nonzero_p95_ms"]
for row in baseline_rows),
}
summary = {
"source_commit": subprocess.run(
["git", "-c", f"safe.directory={ROOT}", "rev-parse", "HEAD"],
cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip(),
"protocol": "Seven interleaved 30 m/s UE pairs; 12 s move plus 5 s dwell; 8 effective tile slots. L157 critical wait remains on and apply cap remains 0 ms.",
"conditions": CONDITIONS,
"movement_medians": medians,
"paired_deltas_vs_assembler_world": pairs,
"candidate_gates": candidate_gates,
"runs": rows,
}
(OUT / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n",
encoding="utf-8")
print("[L166Summary] " + json.dumps({"movement_medians": medians,
"candidate_gates": candidate_gates,
"runs": len(rows)}, sort_keys=True), flush=True)
return 0 if all(candidate_gates.values()) else 1
if __name__ == "__main__":
raise SystemExit(main())