standalone: far-survival audit + LOD8 default with explicit far shells; neighbourhood-first viewer

- FarSurvivalAudit: checks every stratum top/bottom boundary (4x4 columns per far tile) and every
  in-view far connector (sections <=128 voxels, 4 radial rays) against the emitted mesh. Falsified:
  removing one tile's shell triangles -> 16 missing, 16 holes, exit 14. Passes at LOD6/7/8.
- Far connectors follow source radial crossings; strata patches emitted at LOD7/8. Default LOD8.
- Viewer: separate near ShapeWorld for the 27 player tiles, full world built on its own thread;
  27/27 tiles hash-identical to the direct build. W1/W8 60 hash fields equal; routes 5642/5642.
Cost, measured by me head to head (8 workers, cold, BelowNormal 0xFF00): LOD8 without the shells
1.17-1.25 s, LOD8 with the shells 1.80 s; payload 92.6 -> 276.7 MiB. The shells are far too
expensive, and the pipelined benchmark whole view is 2.92 s (shape compiled twice): both to fix.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MmSoF64BbRFmk9C3p1VsBJ
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed 2026-10-04 22:51:44 +02:00
1 parent 5602a281b9
commit a40b0454b3
7 files changed
+1350 -17

No files matched your search

+414 -14
View File
@@ -49,6 +49,8 @@ namespace
V3 Cross(V3 A,V3 B){return {A.Y*B.Z-A.Z*B.Y,A.Z*B.X-A.X*B.Z,A.X*B.Y-A.Y*B.X};}
V3 Normalize(V3 A){return Mul(A,1.0f/std::sqrt(std::max(1.0e-20f,Dot(A,A))));}
#include "FarSurvivalAudit.h"
struct FMeshSheetAudit
{
uint64 Components=0;
@@ -337,7 +339,13 @@ namespace
ID3D11RasterizerState* WireRasterizer=nullptr;
std::unique_ptr<FStandaloneOwnerWorld> World;
std::unique_ptr<FStandaloneShapeWorld> ShapeWorld;
std::unique_ptr<FStandaloneShapeWorld> PendingFullShapeWorld;
std::unique_ptr<FStandaloneStreamingGenerator> Streamer;
std::thread FullShapeBuildThread;
std::atomic<bool> bFullShapeBuildFinished{false};
bool bFullShapeBuildSucceeded=false;
std::string FullShapeBuildError;
std::chrono::steady_clock::time_point FullShapeBuildStart{};
FStandaloneMeshCache MeshCache;
std::string DataPath;
bool bHasCameraOverride=false;
@@ -349,6 +357,7 @@ namespace
std::unordered_set<FStandaloneBatchKey,FStandaloneBatchKeyHash> DirtyBatchSet;
std::unordered_map<uint64,FCrossingWork> CrossingWork;
std::vector<FVoxelTileKey> DesiredTiles;
std::vector<FVoxelTileKey> InitialNeighborhoodTiles;
int32 DesiredByLevel[MaxLevels+1]{};
uint64 GeneratedByLevel[MaxLevels+1]{};
uint64 SamplesByLevel[MaxLevels+1]{};
@@ -362,6 +371,8 @@ namespace
bool bRoundTunnels=false;
bool bRoundTunnelEnhanced=true;
bool bShapeWorld=false;
bool bShapePipelineActive=false;
bool bNeighborhoodReadyReported=false;
bool bShapeAuditOnly=false;
bool bShapeCompare=false;
bool bFarShapeStagePending=false;
@@ -386,6 +397,7 @@ namespace
bool bForceGenerate=false;
std::string CapturePath;
std::string MeshCachePath;
std::chrono::steady_clock::time_point ColdProcessStart{};
std::chrono::steady_clock::time_point GenerationStart{};
bool bShowLevel[MaxLevels+1]={true,true,true,true,true,true,true,true,true};
bool Keys[256]{};
@@ -435,6 +447,15 @@ namespace
(void)bCancel;
if(Streamer)Streamer->Stop();
Streamer.reset();
if(FullShapeBuildThread.joinable())FullShapeBuildThread.join();
PendingFullShapeWorld.reset();
bFullShapeBuildFinished.store(false,std::memory_order_relaxed);
bFullShapeBuildSucceeded=false;
FullShapeBuildError.clear();
InitialNeighborhoodTiles.clear();
bShapePipelineActive=false;
bNeighborhoodReadyReported=false;
bFarShapeStagePending=false;
bGenerationRunning=false;
}
@@ -653,42 +674,95 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
std::printf("Standalone max clip level override applied after world reload: max_level=%d.\n",
MaxClipLevelOverride);
}
if(bShapeWorld&&!BuildShapeWorldForCamera(Error))return false;
if(bShapeWorld&&!BuildShapeWorldForCamera(Error,false,true))return false;
LoadMeshCache();
return true;
}
bool BuildShapeWorldForCamera(std::string& Error,bool bIncludeFarWorld=false)
bool BuildShapeWorldForCamera(std::string& Error,bool bIncludeFarWorld=false,
bool bNeighborhoodOnly=false)
{
const FVector CameraVoxels(Position.X*4.0,Position.Y*4.0,Position.Z*4.0);
const FStandaloneDesiredTileSet Desired=StandaloneWorldMesher::BuildDesiredSet(*World,CameraVoxels);
FIntVector Min(INT32_MAX,INT32_MAX,INT32_MAX),Max(INT32_MIN,INT32_MIN,INT32_MIN);
FIntVector ClipMin(INT32_MAX,INT32_MAX,INT32_MAX),ClipMax(INT32_MIN,INT32_MIN,INT32_MIN);
uint64 FineTiles=0;
InitialNeighborhoodTiles.clear();
const FIntVector CenterChunk=Desired.CenterChunk;
for(const FVoxelTileKey& Tile:Desired.Tiles)
{
const FIntVector Origin=Tile.OriginVoxels();
const int32 Extent=Tile.ExtentVoxels();
ClipMin.X=std::min(ClipMin.X,Origin.X);ClipMin.Y=std::min(ClipMin.Y,Origin.Y);ClipMin.Z=std::min(ClipMin.Z,Origin.Z);
ClipMax.X=std::max(ClipMax.X,Origin.X+Extent);ClipMax.Y=std::max(ClipMax.Y,Origin.Y+Extent);ClipMax.Z=std::max(ClipMax.Z,Origin.Z+Extent);
if(Tile.Level>=World->FullResClipLevels)continue;
if(bNeighborhoodOnly)
{
if(Tile.Level!=0||std::abs(Tile.Coord.X-CenterChunk.X)>1
||std::abs(Tile.Coord.Y-CenterChunk.Y)>1
||std::abs(Tile.Coord.Z-CenterChunk.Z)>1)continue;
InitialNeighborhoodTiles.push_back(Tile);
}
else if(Tile.Level>=World->FullResClipLevels)continue;
Min.X=std::min(Min.X,Origin.X);Min.Y=std::min(Min.Y,Origin.Y);Min.Z=std::min(Min.Z,Origin.Z);
Max.X=std::max(Max.X,Origin.X+Extent);Max.Y=std::max(Max.Y,Origin.Y+Extent);Max.Z=std::max(Max.Z,Origin.Z+Extent);
++FineTiles;
}
if(!FineTiles){Error="no full-detail tiles are available for shape compilation";return false;}
if(bNeighborhoodOnly)
{
if(InitialNeighborhoodTiles.size()!=27)
{Error="the 3x3x3 player neighborhood does not contain 27 LOD0 tiles";return false;}
constexpr int32 NearBuildPad=64;
Min.X-=NearBuildPad;Min.Y-=NearBuildPad;Min.Z-=NearBuildPad;
Max.X+=NearBuildPad;Max.Y+=NearBuildPad;Max.Z+=NearBuildPad;
}
ShapeWorld=std::make_unique<FStandaloneShapeWorld>();
if(!ShapeWorld->Build(*World,Min,Max,ClipMin,ClipMax,WorkerCount,
const int32 ShapeWorkers=bNeighborhoodOnly?(WorkerCount>=8?2:1):WorkerCount;
if(!ShapeWorld->Build(*World,Min,Max,ClipMin,ClipMax,ShapeWorkers,
ShapeDetailVariant,bIncludeFarWorld,Error))
{ShapeWorld.reset();return false;}
FStandaloneShapeWorldAccess::SetActive(ShapeWorld.get());
bFarShapeStagePending=bShapeWorld&&!ShapeWorld->HasFarWorld();
bShapePipelineActive=bNeighborhoodOnly;
bNeighborhoodReadyReported=false;
bFarShapeStagePending=bShapeWorld&&!bNeighborhoodOnly&&!ShapeWorld->HasFarWorld();
std::printf("[ShapeRegion] fine_tiles=%llu bounds=(%d,%d,%d)..(%d,%d,%d) clip=(%d,%d,%d)..(%d,%d,%d)\n",
static_cast<unsigned long long>(FineTiles),Min.X,Min.Y,Min.Z,Max.X,Max.Y,Max.Z,
ClipMin.X,ClipMin.Y,ClipMin.Z,ClipMax.X,ClipMax.Y,ClipMax.Z);
return true;
}
bool StartFullShapeBuild(const std::vector<FVoxelTileKey>& FullDesired,std::string& Error)
{
FIntVector FineMin(INT32_MAX,INT32_MAX,INT32_MAX),FineMax(INT32_MIN,INT32_MIN,INT32_MIN);
FIntVector ClipMin(INT32_MAX,INT32_MAX,INT32_MAX),ClipMax(INT32_MIN,INT32_MIN,INT32_MIN);
for(const FVoxelTileKey& Tile:FullDesired)
{
const FIntVector Origin=Tile.OriginVoxels();const int32 Extent=Tile.ExtentVoxels();
ClipMin.X=std::min(ClipMin.X,Origin.X);ClipMin.Y=std::min(ClipMin.Y,Origin.Y);ClipMin.Z=std::min(ClipMin.Z,Origin.Z);
ClipMax.X=std::max(ClipMax.X,Origin.X+Extent);ClipMax.Y=std::max(ClipMax.Y,Origin.Y+Extent);ClipMax.Z=std::max(ClipMax.Z,Origin.Z+Extent);
if(Tile.Level>=World->FullResClipLevels)continue;
FineMin.X=std::min(FineMin.X,Origin.X);FineMin.Y=std::min(FineMin.Y,Origin.Y);FineMin.Z=std::min(FineMin.Z,Origin.Z);
FineMax.X=std::max(FineMax.X,Origin.X+Extent);FineMax.Y=std::max(FineMax.Y,Origin.Y+Extent);FineMax.Z=std::max(FineMax.Z,Origin.Z+Extent);
}
if(FineMin.X==INT32_MAX||ClipMin.X==INT32_MAX)
{Error="the full shape build has no desired detail or clipmap tiles";return false;}
PendingFullShapeWorld=std::make_unique<FStandaloneShapeWorld>();
FullShapeBuildError.clear();bFullShapeBuildSucceeded=false;
bFullShapeBuildFinished.store(false,std::memory_order_relaxed);
const int32 FullWorkers=WorkerCount>=8?6:WorkerCount;
FullShapeBuildStart=std::chrono::steady_clock::now();
FullShapeBuildThread=std::thread([this,FineMin,FineMax,ClipMin,ClipMax,FullWorkers]()
{
bFullShapeBuildSucceeded=PendingFullShapeWorld->Build(*World,FineMin,FineMax,
ClipMin,ClipMax,FullWorkers,ShapeDetailVariant,true,FullShapeBuildError);
bFullShapeBuildFinished.store(true,std::memory_order_release);
});
std::printf("[ShapePipelineBuildStart] neighborhood_tiles=%zu near_mesh_workers=%d full_shape_workers=%d\n",
InitialNeighborhoodTiles.size(),WorkerCount>=8?2:1,FullWorkers);
std::fflush(stdout);
return true;
}
FStandaloneMeshCacheMetadata MakeMeshCacheMetadata()const
{
FStandaloneMeshCacheMetadata Metadata;
@@ -752,10 +826,19 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
DesiredTiles=Desired.Tiles;
for(int32 I=0;I<=MaxLevels;++I)DesiredByLevel[I]=0;
for(const FVoxelTileKey& Tile:DesiredTiles)++DesiredByLevel[Tile.Level];
Streamer=std::make_unique<FStandaloneStreamingGenerator>(*World,WorkerCount,false,
const int32 StreamWorkers=bShapePipelineActive?(WorkerCount>=8?2:1):WorkerCount;
Streamer=std::make_unique<FStandaloneStreamingGenerator>(*World,StreamWorkers,false,
(!bShapeWorld&&MeshCache.IsLoaded())?&MeshCache:nullptr,
bFarShapeStagePending?World->FullResClipLevels-1:-1);
if(bShapePipelineActive)Streamer->SetDesiredTileFilter(InitialNeighborhoodTiles);
const FStandaloneStreamingUpdate Update=Streamer->Start(CameraVoxels,{},CrossingIndex);
if(bShapePipelineActive&&Update.DesiredTiles!=static_cast<int32>(InitialNeighborhoodTiles.size()))
{
StopGeneration();
Error="the initial neighborhood streamer did not accept all 27 tiles";
std::fprintf(stderr,"World load failed: %s\n",Error.c_str());
return;
}
FCrossingWork& InitialWork=CrossingWork[CrossingIndex];
InitialWork.SubmittedJobs=Update.NewJobs;InitialWork.PendingJobs=Update.NewJobs;
InitialWork.StartTime=GenerationStart;
@@ -765,6 +848,16 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
std::printf("Streaming seed %d%s%s: %d desired tiles, %d full-detail tiles, %d workers; upload budget %d batches / %.1f ms per frame.\n",
Seed,TunnelMode,bShapeWorld?" [shape intervals]":"",Update.DesiredTiles,
Update.FullResTiles,Streamer->GetWorkerCount(),MaxBatchUploadsPerFrame,BatchUploadBudgetMilliseconds);
if(bShapePipelineActive)
{
std::string ShapeError;
if(!StartFullShapeBuild(DesiredTiles,ShapeError))
{
StopGeneration();
std::fprintf(stderr,"World load failed: %s\n",ShapeError.c_str());
return;
}
}
std::fflush(stdout);
}
@@ -797,6 +890,67 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
std::fflush(stdout);
}
void TryCompleteShapePipeline()
{
if(!bShapePipelineActive||InitialNeighborhoodTiles.empty())return;
bool bNeighborhoodUploaded=true;
for(const FVoxelTileKey& Key:InitialNeighborhoodTiles)
if(!ResidentTiles.contains(Key)){bNeighborhoodUploaded=false;break;}
bNeighborhoodUploaded &= DirtyBatchSet.empty();
if(bNeighborhoodUploaded&&!bNeighborhoodReadyReported)
{
bNeighborhoodReadyReported=true;
const double Elapsed=std::chrono::duration<double>(
std::chrono::steady_clock::now()-ColdProcessStart).count();
std::printf("[NeighborhoodReady] %.3fs after cold process start; tiles=%zu/%zu vertices=%llu indices=%llu\n",
Elapsed,InitialNeighborhoodTiles.size(),InitialNeighborhoodTiles.size(),
static_cast<unsigned long long>(TotalVertices),
static_cast<unsigned long long>(TotalIndices));
std::fflush(stdout);
}
if(!bNeighborhoodReadyReported
||!bFullShapeBuildFinished.load(std::memory_order_acquire))return;
if(FullShapeBuildThread.joinable())FullShapeBuildThread.join();
if(!bFullShapeBuildSucceeded||!PendingFullShapeWorld)
{
bShapePipelineActive=false;bGenerationRunning=false;
std::fprintf(stderr,"[ShapePipeline] full shape build failed: %s\n",
FullShapeBuildError.c_str());
PostMessageA(Window,WM_CLOSE,0,0);
return;
}
const double FullShapeSeconds=std::chrono::duration<double>(
std::chrono::steady_clock::now()-FullShapeBuildStart).count();
std::vector<FVoxelTileKey> ResidentKeys;ResidentKeys.reserve(ResidentTiles.size());
for(const auto& Pair:ResidentTiles)ResidentKeys.push_back(Pair.first);
if(Streamer)Streamer->Stop();
Streamer.reset();
ShapeWorld=std::move(PendingFullShapeWorld);
FStandaloneShapeWorldAccess::SetActive(ShapeWorld.get());
bFarShapeStagePending=false;bShapePipelineActive=false;
const FVector CameraVoxels(Position.X*4.0,Position.Y*4.0,Position.Z*4.0);
Streamer=std::make_unique<FStandaloneStreamingGenerator>(*World,WorkerCount,false,nullptr,-1);
++CrossingIndex;
const FStandaloneStreamingUpdate Update=Streamer->Start(CameraVoxels,ResidentKeys,CrossingIndex);
if(Update.DesiredTiles!=static_cast<int32>(DesiredTiles.size()))
{
Streamer->Stop();Streamer.reset();bGenerationRunning=false;
std::fprintf(stderr,"[ShapePipeline] full streamer accepted %d of %zu desired tiles.\n",
Update.DesiredTiles,DesiredTiles.size());
PostMessageA(Window,WM_CLOSE,0,0);
return;
}
FCrossingWork& Work=CrossingWork[CrossingIndex];
Work.SubmittedJobs=Update.NewJobs;Work.PendingJobs=Update.NewJobs;
Work.StartTime=std::chrono::steady_clock::now();
bGenerationRunning=true;
std::printf("[ShapePipelineFullReady] full_shape=%.3fs retained=%d new_jobs=%d workers=%d; filling outward\n",
FullShapeSeconds,Update.RetainedResidentTiles,Update.NewJobs,WorkerCount);
std::fflush(stdout);
}
void MarkBatchDirty(const FStandaloneBatchKey& Key)
{DirtyBatchSet.insert(Key);}
@@ -1050,6 +1204,7 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
}
ReportCrossingWork();
UploadDirtyBatches();
if(bShapePipelineActive)TryCompleteShapePipeline();
if(bTestOneChunkCrossing&&bCrossingMoveFinished&&!bCrossingClosePosted
&&DirtyBatchSet.empty())
{
@@ -1062,11 +1217,11 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
PostMessageA(Window,WM_CLOSE,0,0);
}
}
if(!bFullDetailReported&&IsFullDetailUploaded())
if(!bShapePipelineActive&&!bFullDetailReported&&IsFullDetailUploaded())
{
bFullDetailReported=true;
const double Elapsed=std::chrono::duration<double>(std::chrono::steady_clock::now()-GenerationStart).count();
std::printf("[FullDetailReady] %.3f s after spawn; resident=%zu/%d tiles through LOD%d.\n",
const double Elapsed=std::chrono::duration<double>(std::chrono::steady_clock::now()-ColdProcessStart).count();
std::printf("[FullDetailReady] %.3f s after cold process start; resident=%zu/%d tiles through LOD%d.\n",
Elapsed,ResidentTiles.size(),Streamer->FullResDesiredCount(),World->FullResClipLevels-1);
ReportTileTimings();std::fflush(stdout);
StartFarShapeGeneration();
@@ -1074,8 +1229,8 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
if(!bInitialClipmapReported&&ResidentTiles.size()==DesiredTiles.size()&&DirtyBatchSet.empty())
{
bInitialClipmapReported=true;
const double Elapsed=std::chrono::duration<double>(std::chrono::steady_clock::now()-GenerationStart).count();
std::printf("[InitialClipmapReady] %.3f s after spawn; %zu tiles, %llu vertices, %llu indices, %zu GPU batches.\n",
const double Elapsed=std::chrono::duration<double>(std::chrono::steady_clock::now()-ColdProcessStart).count();
std::printf("[InitialClipmapReady] %.3f s after cold process start; %zu tiles, %llu vertices, %llu indices, %zu GPU batches.\n",
Elapsed,ResidentTiles.size(),static_cast<unsigned long long>(TotalVertices),
static_cast<unsigned long long>(TotalIndices),StreamingBatches.size());
ReportTileTimings();std::fflush(stdout);
@@ -1709,12 +1864,225 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
return Hash;
}
int RunPipelinedStreamingBenchmark(FStandaloneOwnerWorld& World,const FVector& CameraPositionVoxels,
int32 WorkerCount,int32 MaxLevel,FStandaloneShapeWorld::EDetailVariant DetailVariant,
bool bHashDensity,bool bSkipUvAndColors,bool bSkipMaterialGroups,bool bSkipUV,bool bSkipColors,
std::chrono::steady_clock::time_point ColdStart)
{
using Clock=std::chrono::steady_clock;
FStandaloneDesiredTileSet Desired=StandaloneWorldMesher::BuildDesiredSet(World,CameraPositionVoxels);
if(MaxLevel>=0)
Desired.Tiles.erase(std::remove_if(Desired.Tiles.begin(),Desired.Tiles.end(),
[&](const FVoxelTileKey& Tile){return Tile.Level>MaxLevel;}),Desired.Tiles.end());
FIntVector FineMin(INT32_MAX,INT32_MAX,INT32_MAX),FineMax(INT32_MIN,INT32_MIN,INT32_MIN);
FIntVector ClipMin(INT32_MAX,INT32_MAX,INT32_MAX),ClipMax(INT32_MIN,INT32_MIN,INT32_MIN);
for(const FVoxelTileKey& Tile:Desired.Tiles)
{
const FIntVector Origin=Tile.OriginVoxels();const int32 Extent=Tile.ExtentVoxels();
ClipMin.X=std::min(ClipMin.X,Origin.X);ClipMin.Y=std::min(ClipMin.Y,Origin.Y);ClipMin.Z=std::min(ClipMin.Z,Origin.Z);
ClipMax.X=std::max(ClipMax.X,Origin.X+Extent);ClipMax.Y=std::max(ClipMax.Y,Origin.Y+Extent);ClipMax.Z=std::max(ClipMax.Z,Origin.Z+Extent);
if(Tile.Level<World.FullResClipLevels)
{
FineMin.X=std::min(FineMin.X,Origin.X);FineMin.Y=std::min(FineMin.Y,Origin.Y);FineMin.Z=std::min(FineMin.Z,Origin.Z);
FineMax.X=std::max(FineMax.X,Origin.X+Extent);FineMax.Y=std::max(FineMax.Y,Origin.Y+Extent);FineMax.Z=std::max(FineMax.Z,Origin.Z+Extent);
}
}
if(FineMin.X==INT32_MAX||ClipMin.X==INT32_MAX)
{std::fprintf(stderr,"[StreamPipeline] no desired detail or clipmap tiles.\n");return 7;}
const FIntVector CenterChunk=Desired.CenterChunk;
std::vector<FVoxelTileKey> Neighborhood;
FIntVector NearMin(INT32_MAX,INT32_MAX,INT32_MAX),NearMax(INT32_MIN,INT32_MIN,INT32_MIN);
for(const FVoxelTileKey& Tile:Desired.Tiles)
{
if(Tile.Level!=0||std::abs(Tile.Coord.X-CenterChunk.X)>1
||std::abs(Tile.Coord.Y-CenterChunk.Y)>1||std::abs(Tile.Coord.Z-CenterChunk.Z)>1)continue;
Neighborhood.push_back(Tile);const FIntVector Origin=Tile.OriginVoxels();const int32 Extent=Tile.ExtentVoxels();
NearMin.X=std::min(NearMin.X,Origin.X);NearMin.Y=std::min(NearMin.Y,Origin.Y);NearMin.Z=std::min(NearMin.Z,Origin.Z);
NearMax.X=std::max(NearMax.X,Origin.X+Extent);NearMax.Y=std::max(NearMax.Y,Origin.Y+Extent);NearMax.Z=std::max(NearMax.Z,Origin.Z+Extent);
}
if(Neighborhood.empty())
{std::fprintf(stderr,"[StreamPipeline] the player neighborhood is absent from the desired set.\n");return 7;}
constexpr int32 NearBuildPad=64;
NearMin.X-=NearBuildPad;NearMin.Y-=NearBuildPad;NearMin.Z-=NearBuildPad;
NearMax.X+=NearBuildPad;NearMax.Y+=NearBuildPad;NearMax.Z+=NearBuildPad;
const int32 NearWorkers=WorkerCount>=8?2:1;
const int32 FullBuildWorkers=WorkerCount>=8?6:1;
auto NearShape=std::make_unique<FStandaloneShapeWorld>();
std::string BuildError;
const auto NearBuildStart=Clock::now();
if(!NearShape->Build(World,NearMin,NearMax,ClipMin,ClipMax,NearWorkers,
DetailVariant,false,BuildError))
{std::fprintf(stderr,"[StreamPipeline] near shape build failed: %s\n",BuildError.c_str());return 2;}
const double NearShapeSeconds=std::chrono::duration<double>(Clock::now()-NearBuildStart).count();
FStandaloneShapeWorldAccess::SetActive(NearShape.get());
FStandaloneStreamingGenerator NearStreamer(World,NearWorkers,bHashDensity,nullptr,MaxLevel,
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors);
NearStreamer.SetDesiredTileFilter(Neighborhood);
const auto NearStreamStart=Clock::now();
const FStandaloneStreamingUpdate NearUpdate=NearStreamer.Start(CameraPositionVoxels);
if(NearUpdate.DesiredTiles!=static_cast<int32>(Neighborhood.size()))
{NearStreamer.Stop();FStandaloneShapeWorldAccess::SetActive(nullptr);
std::fprintf(stderr,"[StreamPipeline] expected %zu player tiles but queued %d.\n",Neighborhood.size(),NearUpdate.DesiredTiles);return 7;}
FStandaloneShapeWorld FullShape;
bool bFullBuildOk=false;std::string FullBuildError;
const auto FullBuildStart=Clock::now();
std::thread FullBuildThread([&]
{bFullBuildOk=FullShape.Build(World,FineMin,FineMax,ClipMin,ClipMax,FullBuildWorkers,
DetailVariant,true,FullBuildError);});
struct FPipelineTileHashes {uint64 Density=0,Mesh=0,Payload=0,Topology=0;};
std::unordered_map<FVoxelTileKey,FPipelineTileHashes,FStandaloneTileKeyHash> NearHashByKey;
NearHashByKey.reserve(Neighborhood.size()*2+1);
std::vector<FHashedTile> DensityRows,MeshRows,PayloadRows,TopologyRows;
std::vector<FStandaloneCpuTile> NearTiles;NearTiles.reserve(Neighborhood.size());
std::chrono::steady_clock::time_point NeighborhoodReady{};
int32 NearFinished=0;
while(NearFinished<NearUpdate.DesiredTiles)
{
std::vector<FStandaloneCpuTile> Ready=NearStreamer.TakeReady();
if(Ready.empty()){std::this_thread::sleep_for(std::chrono::milliseconds(1));continue;}
for(FStandaloneCpuTile& Tile:Ready)
{
++NearFinished;
if(bHashDensity)
{
if(Tile.bHasDensityHash)DensityRows.push_back({Tile.Key,Tile.DensityHash});
if(Tile.bHasMeshHash)MeshRows.push_back({Tile.Key,Tile.MeshHash});
if(Tile.bHasPayloadHash)PayloadRows.push_back({Tile.Key,Tile.PayloadHash});
if(Tile.bHasTopologyHash)TopologyRows.push_back({Tile.Key,Tile.TopologyHash});
NearHashByKey[Tile.Key]={Tile.DensityHash,Tile.MeshHash,Tile.PayloadHash,Tile.TopologyHash};
}
NeighborhoodReady=std::max(NeighborhoodReady,Tile.CompletedAt);
NearTiles.push_back(std::move(Tile));
}
}
NearStreamer.Stop();
const double NearStreamSeconds=std::chrono::duration<double>(NeighborhoodReady-NearStreamStart).count();
FStandaloneShapeWorldAccess::SetActive(nullptr);
// Release the small shape before the large build finishes allocating its packed field.
NearTiles.clear();NearTiles.shrink_to_fit();NearShape.reset();
FullBuildThread.join();
const double FullShapeSeconds=std::chrono::duration<double>(Clock::now()-FullBuildStart).count();
if(!bFullBuildOk)
{std::fprintf(stderr,"[StreamPipeline] full shape build failed: %s\n",FullBuildError.c_str());return 2;}
FStandaloneShapeWorldAccess::SetActive(&FullShape);
std::unordered_set<FVoxelTileKey,FStandaloneTileKeyHash> NearSet;
NearSet.reserve(Neighborhood.size()*2+1);for(const FVoxelTileKey& Key:Neighborhood)NearSet.insert(Key);
std::vector<FVoxelTileKey> Remaining;Remaining.reserve(Desired.Tiles.size()-Neighborhood.size());
int32 FullDetailExpected=0;
for(const FVoxelTileKey& Tile:Desired.Tiles)
{
if(Tile.Level<World.FullResClipLevels)++FullDetailExpected;
if(!NearSet.contains(Tile))Remaining.push_back(Tile);
}
FStandaloneStreamingGenerator FullStreamer(World,WorkerCount,bHashDensity,nullptr,MaxLevel,
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors);
FullStreamer.SetDesiredTileFilter(Remaining);
const auto FillStart=Clock::now();
const FStandaloneStreamingUpdate FillUpdate=FullStreamer.Start(CameraPositionVoxels);
int32 Finished=0,FullDetailFinished=static_cast<int32>(Neighborhood.size());
const int32 FullDetailRemaining=std::max(0,FullDetailExpected-FullDetailFinished);
std::chrono::steady_clock::time_point FullDetailReady=NeighborhoodReady,WholeViewReady{};
bool bFullDetailReported=false;
while(Finished<FillUpdate.DesiredTiles)
{
std::vector<FStandaloneCpuTile> Ready=FullStreamer.TakeReady();
if(Ready.empty()){std::this_thread::sleep_for(std::chrono::milliseconds(1));continue;}
for(const FStandaloneCpuTile& Tile:Ready)
{
++Finished;WholeViewReady=std::max(WholeViewReady,Tile.CompletedAt);
if(Tile.Key.Level<World.FullResClipLevels)
{++FullDetailFinished;FullDetailReady=std::max(FullDetailReady,Tile.CompletedAt);}
if(bHashDensity)
{
if(Tile.bHasDensityHash)DensityRows.push_back({Tile.Key,Tile.DensityHash});
if(Tile.bHasMeshHash)MeshRows.push_back({Tile.Key,Tile.MeshHash});
if(Tile.bHasPayloadHash)PayloadRows.push_back({Tile.Key,Tile.PayloadHash});
if(Tile.bHasTopologyHash)TopologyRows.push_back({Tile.Key,Tile.TopologyHash});
}
}
if(!bFullDetailReported&&FullDetailRemaining>0&&FullDetailFinished>=FullDetailExpected)
{
std::printf("[PipelineMilestone] full_detail_ready=%.3fs generated=%d/%d\n",
std::chrono::duration<double>(FullDetailReady-ColdStart).count(),
FullDetailFinished,FullDetailExpected);
bFullDetailReported=true;
}
}
FullStreamer.Stop();
const double WholeViewSeconds=std::chrono::duration<double>(WholeViewReady-ColdStart).count();
const double NeighborhoodSeconds=std::chrono::duration<double>(NeighborhoodReady-ColdStart).count();
const double FullDetailSeconds=std::chrono::duration<double>(FullDetailReady-ColdStart).count();
bool bNearParityPassing=true;
if(bHashDensity)
{
FStandaloneStreamingGenerator NearVerifier(World,NearWorkers,true,nullptr,MaxLevel,
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors);
NearVerifier.SetDesiredTileFilter(Neighborhood);
const FStandaloneStreamingUpdate VerifyUpdate=NearVerifier.Start(CameraPositionVoxels);
int32 VerifiedTiles=0,NearHashMismatches=0;
while(VerifiedTiles<VerifyUpdate.DesiredTiles)
{
std::vector<FStandaloneCpuTile> Ready=NearVerifier.TakeReady();
if(Ready.empty()){std::this_thread::sleep_for(std::chrono::milliseconds(1));continue;}
for(const FStandaloneCpuTile& Tile:Ready)
{
++VerifiedTiles;
const auto It=NearHashByKey.find(Tile.Key);
if(It==NearHashByKey.end()||!Tile.bHasDensityHash||!Tile.bHasMeshHash
||!Tile.bHasPayloadHash||!Tile.bHasTopologyHash
||It->second.Density!=Tile.DensityHash||It->second.Mesh!=Tile.MeshHash
||It->second.Payload!=Tile.PayloadHash||It->second.Topology!=Tile.TopologyHash)
{
++NearHashMismatches;
if(NearHashMismatches<=4)
std::fprintf(stderr,"[PipelineNeighborhoodHashMismatch] key=(%d,%d,%d,lod%d)\n",
Tile.Key.Coord.X,Tile.Key.Coord.Y,Tile.Key.Coord.Z,Tile.Key.Level);
}
}
}
NearVerifier.Stop();
bNearParityPassing=VerifiedTiles==static_cast<int32>(Neighborhood.size())
&&NearHashMismatches==0;
std::printf("[PipelineNeighborhoodParity] tiles=%d/%zu hash_fields=%d mismatched_tiles=%d status=%s\n",
VerifiedTiles,Neighborhood.size(),VerifiedTiles*4,NearHashMismatches,
bNearParityPassing?"PASS":"FAIL");
}
std::printf("[StreamPipeline] workers=%d near_mesh_workers=%d full_shape_workers=%d near_shape=%.3fs near_mesh=%.3fs full_shape=%.3fs neighborhood_ready=%.3fs full_detail_ready=%.3fs whole_view_ready=%.3fs neighborhood_tiles=%zu/%zu full_detail_tiles=%d/%d whole_tiles=%d/%zu cache_hits=0 full_payload=1\n",
WorkerCount,NearWorkers,FullBuildWorkers,NearShapeSeconds,NearStreamSeconds,FullShapeSeconds,
NeighborhoodSeconds,FullDetailSeconds,WholeViewSeconds,Neighborhood.size(),Neighborhood.size(),
FullDetailFinished,FullDetailExpected,static_cast<int32>(Desired.Tiles.size()),Desired.Tiles.size());
if(bHashDensity)
{
std::vector<FHashedTile> FullDensity,FullMesh,FullPayload,FullTopology;
for(const FHashedTile& Row:DensityRows)if(Row.Key.Level<World.FullResClipLevels)FullDensity.push_back(Row);
for(const FHashedTile& Row:MeshRows)if(Row.Key.Level<World.FullResClipLevels)FullMesh.push_back(Row);
for(const FHashedTile& Row:PayloadRows)if(Row.Key.Level<World.FullResClipLevels)FullPayload.push_back(Row);
for(const FHashedTile& Row:TopologyRows)if(Row.Key.Level<World.FullResClipLevels)FullTopology.push_back(Row);
std::printf("[PipelineFullDetailHash] tiles=%d density=0x%016llx mesh=0x%016llx payload=0x%016llx topology=0x%016llx\n",
FullDetailExpected,static_cast<unsigned long long>(HashTileRows(FullDensity)),
static_cast<unsigned long long>(HashTileRows(FullMesh)),
static_cast<unsigned long long>(HashTileRows(FullPayload)),
static_cast<unsigned long long>(HashTileRows(FullTopology)));
std::printf("[PipelineWholeViewHash] tiles=%zu density=0x%016llx mesh=0x%016llx payload=0x%016llx topology=0x%016llx\n",
Desired.Tiles.size(),static_cast<unsigned long long>(HashTileRows(DensityRows)),
static_cast<unsigned long long>(HashTileRows(MeshRows)),
static_cast<unsigned long long>(HashTileRows(PayloadRows)),
static_cast<unsigned long long>(HashTileRows(TopologyRows)));
}
FStandaloneShapeWorldAccess::SetActive(nullptr);
return FullDetailFinished==FullDetailExpected&&Finished==FillUpdate.DesiredTiles
&&bNearParityPassing?0:6;
}
int RunStreamingBenchmark(FStandaloneOwnerWorld& World,const FVector& CameraPositionVoxels,
int32 WorkerCount,int32 MaxLevel,bool bRoundTunnels,
bool bEnhancedWalls,const FStandaloneMeshCache* Cache,
bool bHashDensity,bool bSkipUvAndColors,bool bSkipMaterialGroups,
bool bSkipUV,bool bSkipColors,
bool bShardAudit,float CameraYaw,float CameraPitch,
bool bShardAudit,bool bFarSurvivalAudit,bool bFarAuditBadShell,
float CameraYaw,float CameraPitch,
const std::string& BakeCachePath,
const FStandaloneMeshCacheMetadata& CacheMetadata)
{
@@ -1731,6 +2099,14 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
const int32 FullDetailExpected=static_cast<int32>(std::count_if(
Desired.Tiles.begin(),Desired.Tiles.end(),[&](const FVoxelTileKey& Tile)
{return Tile.Level<World.FullResClipLevels;}));
std::unique_ptr<FFarSurvivalAudit> FarAudit;
if(bFarSurvivalAudit)
{
const FStandaloneShapeWorld* Shape=FStandaloneShapeWorldAccess::GetActive();
if(!Shape||!Shape->HasFarWorld())
{std::fprintf(stderr,"Far survival audit requires an active shape world with far-parent data.\n");return 15;}
FarAudit=std::make_unique<FFarSurvivalAudit>(World,*Shape,Desired.Tiles,bFarAuditBadShell);
}
std::array<std::vector<FHashedTile>,MaxLevels+1> HashRows;
std::array<std::vector<FHashedTile>,MaxLevels+1> MeshHashRows;
std::array<std::vector<FHashedTile>,MaxLevels+1> PayloadHashRows;
@@ -1787,6 +2163,7 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
}
for(FStandaloneCpuTile& Tile:Ready)
{
if(FarAudit)FarAudit->AuditTile(Tile);
++Finished;
const int32 Level=Tile.Key.Level;
if(PlayerNeighborhoodTiles.contains(Tile.Key))
@@ -1960,6 +2337,12 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
}
if(Finished!=DesiredTotal||FullDetailFinished!=FullDetailExpected)return 6;
if(bShardAudit&&(MeshSheetStats.ThinSheets!=0||MeshSheetStats.SmallComponentsInView!=0))return 13;
if(FarAudit)
{
FarAudit->Finalize();
FarAudit->Print();
if(!FarAudit->Passing())return 14;
}
return 0;
}
@@ -2038,7 +2421,8 @@ float3 Lit=Rock*(Ambient+0.88*Diffuse)+float3(0.50,0.69,0.82)*Head*0.75;
int main(int Argc,char** Argv)
{
FApp App;App.DataPath=DefaultDataPath().string();
const auto ColdProcessStart=std::chrono::steady_clock::now();
FApp App;App.DataPath=DefaultDataPath().string();App.ColdProcessStart=ColdProcessStart;
std::string JoinedCommandLine;
for(int32 I=1;I<Argc;++I)
{if(!JoinedCommandLine.empty())JoinedCommandLine.push_back(' ');JoinedCommandLine+=Argv[I];}
@@ -2047,8 +2431,11 @@ int main(int Argc,char** Argv)
int32 MaxClipLevelOverride=-1;
int32 ShapeCompareLevel=-1,ShapeCompareTileLimit=0;
bool bStreamBenchmark=false;
bool bStreamPipeline=false;
bool bProfileDetailBand=false;
bool bShardAudit=false;
bool bFarSurvivalAudit=false;
bool bFarAuditBadShell=false;
bool bStreamHash=false;
bool bSkipUvAndColors=false;
bool bSkipUV=false;
@@ -2085,6 +2472,8 @@ int main(int Argc,char** Argv)
else if(Arg=="--round-tunnels-legacy")
{App.bRoundTunnels=true;App.bRoundTunnelEnhanced=false;}
else if(Arg=="--stream-benchmark")bStreamBenchmark=true;
else if(Arg=="--stream-pipeline")
{bStreamPipeline=true;bStreamBenchmark=true;App.bShapeWorld=true;App.bForceGenerate=true;}
else if(Arg=="--edit-benchmark")
{bEditBenchmark=true;bStreamBenchmark=true;App.bShapeWorld=true;}
else if(Arg=="--stream-hash")bStreamHash=true;
@@ -2093,6 +2482,9 @@ int main(int Argc,char** Argv)
else if(Arg=="--skip-colors")bSkipColors=true;
else if(Arg=="--skip-material-groups")bSkipMaterialGroups=true;
else if(Arg=="--shard-audit")bShardAudit=true;
else if(Arg=="--far-survival-audit")bFarSurvivalAudit=true,App.bShapeWorld=true;
else if(Arg=="--far-audit-bad-shell")
{bFarAuditBadShell=true;bFarSurvivalAudit=true;App.bShapeWorld=true;}
else if(Arg=="--compare-round-walls")
{bRoundWallCompare=true;App.bRoundTunnels=true;}
else if(Arg.starts_with("--round-compare-tiles="))
@@ -2166,6 +2558,7 @@ int main(int Argc,char** Argv)
:(App.bShapeCompare?ShapeCompareLevel>=App.World->FullResClipLevels
:(BenchmarkLevel>=App.World->FullResClipLevels));
if(App.bShapeWorld&&(App.bShapeAuditOnly||App.bShapeCompare||bStreamBenchmark||BenchmarkLevel>=0)
&&!bStreamPipeline
&&!App.BuildShapeWorldForCamera(Error,bNeedFarShapeWorld))
{std::fprintf(stderr,"Shape interval build failed: %s\n",Error.c_str());return 2;}
if(App.bShapeAuditOnly)
@@ -2181,6 +2574,12 @@ int main(int Argc,char** Argv)
return RunRoundWallComparison(*App.World,App.Position,RoundCompareTileCount);
if(bStreamBenchmark)
{
if(bStreamPipeline)
return RunPipelinedStreamingBenchmark(*App.World,
FVector(App.Position.X*4.0,App.Position.Y*4.0,App.Position.Z*4.0),
App.WorkerCount,StreamBenchmarkMaxLevel,App.ShapeDetailVariant,
bStreamHash||!BakeMeshCachePath.empty(),bSkipUvAndColors,bSkipMaterialGroups,
bSkipUV,bSkipColors,ColdProcessStart);
if(bEditBenchmark)
return RunEditTransactionBenchmark(*App.World,*App.ShapeWorld,
FVector(App.Position.X*4.0,App.Position.Y*4.0,App.Position.Z*4.0),App.WorkerCount);
@@ -2194,7 +2593,8 @@ int main(int Argc,char** Argv)
App.WorkerCount,StreamBenchmarkMaxLevel,App.bRoundTunnels,
App.bRoundTunnelEnhanced,(!App.bShapeWorld&&App.MeshCache.IsLoaded())?&App.MeshCache:nullptr,
bStreamHash,bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors,bShardAudit,
App.Yaw,App.Pitch,BakeMeshCachePath,App.MakeMeshCacheMetadata());
bFarSurvivalAudit,bFarAuditBadShell,App.Yaw,App.Pitch,
BakeMeshCachePath,App.MakeMeshCacheMetadata());
}
if(BenchmarkLevel>=0)
return RunTileBenchmark(*App.World,App.Position,BenchmarkLevel,BenchmarkCount,