Fr0zka/VoxelForge
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
32b86708334438863a3e165b1d8ee06c4d69cfc1
VoxelForge/Standalone/Source/StreamBenchmarkPipeline.h
T
Fr0zkaandClaude Opus 5.5 32b8670833 wip(assembler): round L12 -- region pages keep LOD1 block proofs: 24 m crossing 5x faster 
Analytic region pages now retain the immutable shared manifest, so LOD1 page tiles get the L9
uniform-block proofs (71.7 -> 5.6 ms per tile). Reviewer W12 no-hash: 24 m crossing 0.898 ->
0.166 s; Luna: 1 km 5.89 -> 1.25 s. Spawn unchanged; all L1/FloatingIslands hashes and the 24 m
crossing hash unchanged (W1/W12).
KNOWN BUG (pre-existing, measured here): the same tile key yields different density from the base
shape (manifest field) than from a region page (legacy field), sometimes different mesh too, so
output depends on streaming history. To be fixed in L13.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CSFYgsdW8ejZSyiMf4qRSX
2026-10-20 01:07:47 +02:00

1109 lines
67 KiB
C++
Raw Blame History

#pragma once
#include "OwnerWorldLoader.h"
#include "StreamingWorld.h"
#include "LinuxScreenSpaceGate.h"
#include <algorithm>
#include <array>
#include <chrono>
#include <cmath>
#include <cstdint>
#include <ctime>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <future>
#include <limits>
#include <numeric>
#include <thread>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace StandaloneStreamBenchmark
{
struct FSceneExportWriter
{
std::ofstream File;
bool bGood=true;
std::uint32_t TileCount=0;
template<typename T> void Write(const T& Value)
{
File.write(reinterpret_cast<const char*>(&Value),sizeof(Value));
bGood=bGood&&static_cast<bool>(File);
}
bool Open(const std::filesystem::path& Path,std::uint32_t VoxelSizeMicrons,
const FVector& CameraVoxels,float YawDegrees,float PitchDegrees,
std::uint32_t ExpectedTiles)
{
std::error_code Error;
if(!Path.parent_path().empty())
std::filesystem::create_directories(Path.parent_path(),Error);
if(Error)return false;
File.open(Path,std::ios::binary|std::ios::trunc);
if(!File)return false;
constexpr char Magic[8]={'V','F','M','S','H','0','0','1'};
File.write(Magic,sizeof(Magic));
const std::uint32_t Version=1;
Write(Version);Write(VoxelSizeMicrons);Write(ExpectedTiles);
const double MicronsToMeters=static_cast<double>(VoxelSizeMicrons)/1000000.0;
const double Camera[3]={CameraVoxels.X*MicronsToMeters,
CameraVoxels.Y*MicronsToMeters,CameraVoxels.Z*MicronsToMeters};
for(const double Value:Camera)Write(Value);
const double Yaw=YawDegrees,Pitch=PitchDegrees;
Write(Yaw);Write(Pitch);
return bGood;
}
void Add(const FStandaloneCpuTile& Tile)
{
const std::int32_t Key[4]={Tile.Key.Coord.X,Tile.Key.Coord.Y,
Tile.Key.Coord.Z,Tile.Key.Level};
for(const auto Value:Key)Write(Value);
const std::uint32_t SurfaceVertexCount=Tile.SurfaceVertexCount;
const std::uint32_t SkirtTriangleCount=Tile.SkirtTriangleCount;
const std::uint32_t VertexCount=static_cast<std::uint32_t>(Tile.Vertices.size());
const std::uint32_t IndexCount=static_cast<std::uint32_t>(Tile.Indices.size());
const std::uint32_t MaskCount=static_cast<std::uint32_t>(Tile.TriangleSkirtFaceMasks.size());
Write(SurfaceVertexCount);Write(SkirtTriangleCount);
Write(VertexCount);Write(IndexCount);Write(MaskCount);
for(const FStandaloneCpuVertex& Vertex:Tile.Vertices)
for(const float Value:Vertex.Position)Write(Value);
for(const FStandaloneCpuVertex& Vertex:Tile.Vertices)
for(const float Value:Vertex.Normal)Write(Value);
if(IndexCount)
File.write(reinterpret_cast<const char*>(Tile.Indices.data()),
static_cast<std::streamsize>(sizeof(std::uint32_t)*Tile.Indices.size()));
if(MaskCount)
File.write(reinterpret_cast<const char*>(Tile.TriangleSkirtFaceMasks.data()),
static_cast<std::streamsize>(Tile.TriangleSkirtFaceMasks.size()));
bGood=bGood&&static_cast<bool>(File);
++TileCount;
}
bool Close(std::uint32_t ExpectedTiles)
{
File.flush();
const bool bComplete=bGood&&static_cast<bool>(File)&&TileCount==ExpectedTiles;
File.close();
return bComplete;
}
};
inline void HashWord(uint64& Hash,uint64 Word,int32 Bytes=8)
{
for(int32 I=0;I<Bytes;++I)
{
Hash^=static_cast<uint8>(Word>>(I*8));
Hash*=1099511628211ull;
}
}
struct FHashedTile
{
FVoxelTileKey Key;
uint64 Hash=14695981039346656037ull;
};
inline bool TileKeyLessForHash(const FVoxelTileKey& A,const FVoxelTileKey& B)
{
if(A.Level!=B.Level)return A.Level<B.Level;
if(A.Coord.Z!=B.Coord.Z)return A.Coord.Z<B.Coord.Z;
if(A.Coord.Y!=B.Coord.Y)return A.Coord.Y<B.Coord.Y;
return A.Coord.X<B.Coord.X;
}
inline uint64 HashTileRows(std::vector<FHashedTile>& Rows)
{
std::sort(Rows.begin(),Rows.end(),[](const FHashedTile& A,const FHashedTile& B)
{return TileKeyLessForHash(A.Key,B.Key);});
uint64 Hash=14695981039346656037ull;
for(const FHashedTile& Row:Rows)
{
HashWord(Hash,static_cast<uint32>(Row.Key.Level),4);
HashWord(Hash,static_cast<uint32>(Row.Key.Coord.X),4);
HashWord(Hash,static_cast<uint32>(Row.Key.Coord.Y),4);
HashWord(Hash,static_cast<uint32>(Row.Key.Coord.Z),4);
HashWord(Hash,Row.Hash);
}
return Hash;
}
inline int RunPipelinedStreamingBenchmark(std::shared_ptr<FStandaloneOwnerWorld> OwnerWorld,
const VoxelForgeCore::ResolvedWorldSnapshot& CoreSnapshot,
const FVector& CameraPositionVoxels,int32 WorkerCount,int32 MaxLevel,
VoxelForgeCore::DetailVariant DetailVariant,const VoxelForgeCore::JobScheduler& Scheduler,
bool bHashDensity,bool bSkipUvAndColors,bool bSkipMaterialGroups,bool bSkipUV,bool bSkipColors,
std::chrono::steady_clock::time_point ColdStart,bool bProfileColumnOverlap,
int32 FarHorizonLOD8InteriorStepVoxels,bool bReportMeshCoverage=false,
const std::filesystem::path& MeshExportPath={},float CameraYawDegrees=0.0f,
float CameraPitchDegrees=0.0f,bool bScreenSpaceGate=false,
bool bUseSharedColumnCache=true,FVector CameraStepVoxels=FVector::ZeroVector,
bool bAuditUniformBlockProofs=false,bool bReportLodWorkProfile=false,
bool bRegionCrossing=false,
bool bAuditTileShapeReadiness=false,int DiagnosticSlowShapeCompileMs=0,
bool bAuditBasePageEquivalence=false)
{
using Clock=std::chrono::steady_clock;
FStandaloneOwnerWorld& World=*OwnerWorld;
const VoxelForgeCore::WorldAssemblyManifest* Assembly=CoreSnapshot.worldAssembly();
std::array<double,9> ClosestDisplayDistances{};
std::array<std::int64_t,9> PixelBudgetsSubvoxels{};
if(bScreenSpaceGate)
{
ClosestDisplayDistances=StandaloneScreenSpaceGate::ClosestDisplayedDistancesMeters(
World,CameraPositionVoxels,World.CoreVoxelSizeMicrons);
const double VoxelMeters=static_cast<double>(World.CoreVoxelSizeMicrons)/1000000.0;
for(int32 Lod=3;Lod<=World.MaxClipLevel&&Lod<9;++Lod)
PixelBudgetsSubvoxels[static_cast<std::size_t>(Lod)]=static_cast<std::int64_t>(std::floor(
ClosestDisplayDistances[static_cast<std::size_t>(Lod)]
*StandaloneScreenSpaceGate::OnePixelRadians/VoxelMeters
*VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel));
}
const bool bRoundTunnels=World.Generator->bStandaloneResearchRoundTunnels;
const bool bEnhancedTunnelWalls=World.Generator->bStandaloneResearchEnhancedWalls;
VoxelForgeCore::World Shape(CoreSnapshot);
FStandaloneDesiredTileSet Desired=StandaloneWorldMesher::BuildDesiredSet(World,CameraPositionVoxels);
const auto CoreTiles=Shape.desiredTiles(
{CameraPositionVoxels.X,CameraPositionVoxels.Y,CameraPositionVoxels.Z});
Desired.Tiles.clear();Desired.Tiles.reserve(CoreTiles.size());
for(const auto& Key:CoreTiles)
Desired.Tiles.emplace_back(FIntVector(Key.coordinate.x,Key.coordinate.y,Key.coordinate.z),Key.lod);
if(MaxLevel>=0)
Desired.Tiles.erase(std::remove_if(Desired.Tiles.begin(),Desired.Tiles.end(),
[&](const FVoxelTileKey& Tile){return Tile.Level>MaxLevel;}),Desired.Tiles.end());
const bool bHasCameraStep=std::abs(CameraStepVoxels.X)+std::abs(CameraStepVoxels.Y)
+std::abs(CameraStepVoxels.Z)>1.0e-6f;
FStandaloneDesiredTileSet MovedDesired;
if(bHasCameraStep)
{
const FVector MovedCamera=CameraPositionVoxels+CameraStepVoxels;
MovedDesired=StandaloneWorldMesher::BuildDesiredSet(World,MovedCamera);
const auto MovedCoreTiles=Shape.desiredTiles({MovedCamera.X,MovedCamera.Y,MovedCamera.Z});
MovedDesired.Tiles.clear();MovedDesired.Tiles.reserve(MovedCoreTiles.size());
for(const auto& Key:MovedCoreTiles)
MovedDesired.Tiles.emplace_back(FIntVector(Key.coordinate.x,Key.coordinate.y,Key.coordinate.z),Key.lod);
if(MaxLevel>=0)
MovedDesired.Tiles.erase(std::remove_if(MovedDesired.Tiles.begin(),MovedDesired.Tiles.end(),
[&](const FVoxelTileKey& Tile){return Tile.Level>MaxLevel;}),MovedDesired.Tiles.end());
}
if(!Desired.Tiles.empty())
{
FIntVector volumeMin(INT32_MAX,INT32_MAX,INT32_MAX);
FIntVector volumeMax(INT32_MIN,INT32_MIN,INT32_MIN);
int32 minimumLod=INT32_MAX,maximumLod=INT32_MIN;
for(const FVoxelTileKey& tile:Desired.Tiles)
{
const FIntVector origin=tile.OriginVoxels();
const int32 extent=tile.ExtentVoxels();
volumeMin.X=std::min(volumeMin.X,origin.X);
volumeMin.Y=std::min(volumeMin.Y,origin.Y);
volumeMin.Z=std::min(volumeMin.Z,origin.Z);
volumeMax.X=std::max(volumeMax.X,origin.X+extent);
volumeMax.Y=std::max(volumeMax.Y,origin.Y+extent);
volumeMax.Z=std::max(volumeMax.Z,origin.Z+extent);
minimumLod=std::min(minimumLod,tile.Level);
maximumLod=std::max(maximumLod,tile.Level);
}
std::printf("[PipelineVolume] tiles=%zu lod=%d..%d min_voxel=(%d,%d,%d) max_voxel=(%d,%d,%d)\n",
Desired.Tiles.size(),minimumLod,maximumLod,volumeMin.X,volumeMin.Y,volumeMin.Z,
volumeMax.X,volumeMax.Y,volumeMax.Z);
}
FIntVector FineMin(INT32_MAX,INT32_MAX,INT32_MAX),FineMax(INT32_MIN,INT32_MIN,INT32_MIN);
FIntVector ClipMin(INT32_MAX,INT32_MAX,INT32_MAX),ClipMax(INT32_MIN,INT32_MIN,INT32_MIN);
const auto AccumulateBounds=[&](const std::vector<FVoxelTileKey>& Tiles)
{
for(const FVoxelTileKey& Tile:Tiles)
{
const FIntVector Origin=Tile.OriginVoxels();const int32 Extent=Tile.ExtentVoxels();
ClipMin.X=std::min(ClipMin.X,Origin.X);ClipMin.Y=std::min(ClipMin.Y,Origin.Y);ClipMin.Z=std::min(ClipMin.Z,Origin.Z);
ClipMax.X=std::max(ClipMax.X,Origin.X+Extent);ClipMax.Y=std::max(ClipMax.Y,Origin.Y+Extent);ClipMax.Z=std::max(ClipMax.Z,Origin.Z+Extent);
if(Tile.Level<World.FullResClipLevels)
{
FineMin.X=std::min(FineMin.X,Origin.X);FineMin.Y=std::min(FineMin.Y,Origin.Y);FineMin.Z=std::min(FineMin.Z,Origin.Z);
FineMax.X=std::max(FineMax.X,Origin.X+Extent);FineMax.Y=std::max(FineMax.Y,Origin.Y+Extent);FineMax.Z=std::max(FineMax.Z,Origin.Z+Extent);
}
}
};
AccumulateBounds(Desired.Tiles);
if(bHasCameraStep&&!bRegionCrossing)AccumulateBounds(MovedDesired.Tiles);
if(FineMin.X==INT32_MAX||ClipMin.X==INT32_MAX)
{std::fprintf(stderr,"[StreamPipeline] no desired detail or clipmap tiles.\n");return 7;}
const FIntVector CenterChunk=Desired.CenterChunk;
std::unordered_set<FVoxelTileKey,FStandaloneTileKeyHash> Neighborhood;
Neighborhood.reserve(32);
for(const FVoxelTileKey& Tile:Desired.Tiles)
if(Tile.Level==0&&std::abs(Tile.Coord.X-CenterChunk.X)<=1
&&std::abs(Tile.Coord.Y-CenterChunk.Y)<=1&&std::abs(Tile.Coord.Z-CenterChunk.Z)<=1)
Neighborhood.insert(Tile);
if(Neighborhood.size()!=27)
{std::fprintf(stderr,"[StreamPipeline] expected 27 player-neighborhood tiles, found %zu.\n",Neighborhood.size());return 7;}
FStandaloneDesiredTileSet ShapeFreeDesired=Desired;
ShapeFreeDesired.Tiles.clear();
std::vector<FVoxelTileKey> ShapeFreeTiles;
ShapeFreeTiles.reserve(Neighborhood.size());
int32 ShapeFreeNeighborhood=0;
for(const FVoxelTileKey& Tile:Desired.Tiles)
{
if(!Neighborhood.contains(Tile))continue;
const VoxelForgeCore::TileKey CoreKey{{Tile.Coord.X,Tile.Coord.Y,Tile.Coord.Z},Tile.Level};
if(!Shape.isTileShapeFree(CoreKey))continue;
ShapeFreeDesired.Tiles.push_back(Tile);
ShapeFreeTiles.push_back(Tile);
if(Neighborhood.contains(Tile))++ShapeFreeNeighborhood;
}
if(bAuditTileShapeReadiness)
{
FVoxelTileKey DeliberatelyWrong(FIntVector(0,0,31250),0);
for(int32 Z=100000;Z<100100;++Z)
{
DeliberatelyWrong=FVoxelTileKey(FIntVector(0,0,Z),0);
if(!Shape.isTileShapeFree({{0,0,Z},0}))break;
}
VoxelForgeCore::TileGenerationOptions NegativeControlOptions;
NegativeControlOptions.diagnosticForceNativeOnly=true;
std::string NegativeControlError;
const auto MisMarked=Shape.generateTile(
{{DeliberatelyWrong.Coord.X,DeliberatelyWrong.Coord.Y,DeliberatelyWrong.Coord.Z},0},
NegativeControlOptions,NegativeControlError);
const bool bGuardCaught=!MisMarked
&&NegativeControlError.find("shape-free guard")!=std::string::npos;
std::printf("[ShapeReadinessNegativeControl] deliberately_mismarked=(%d,%d,%d,lod0) result=%s guard_caught=%d error=%s\n",
DeliberatelyWrong.Coord.X,DeliberatelyWrong.Coord.Y,DeliberatelyWrong.Coord.Z,
MisMarked?"generated":"rejected",bGuardCaught?1:0,
NegativeControlError.c_str());
if(!bGuardCaught)
{std::fprintf(stderr,"[StreamPipeline] shape-free guard did not reject the deliberately mis-marked tile.\n");return 9;}
}
std::printf("[ShapeReadiness] shape_free_desired=%zu/%zu near_ring=%d/27 proof=single_SurfaceWorld_plan_no_features_routes_passages_edits_or_owner_primitive_overlap\n",
ShapeFreeTiles.size(),Desired.Tiles.size(),ShapeFreeNeighborhood);
// The initial desired set now owns all clipmap metadata needed by this Core-only run.
// Release the separate compatibility-world generation graph before compiling Core.
World.Generator.reset();
World.Manager.reset();
World.DefinitionStorage.clear();
World.BiomeStorage.clear();
World.OperationStorage.clear();
World.DensitySamples.clear();
const bool bIncludeFarWorld=MaxLevel<0||MaxLevel>=World.FullResClipLevels;
VoxelForgeCore::ShapeBuildRequest ShapeRequest;
ShapeRequest.minVoxel={FineMin.X,FineMin.Y,FineMin.Z};
ShapeRequest.maxVoxel={FineMax.X,FineMax.Y,FineMax.Z};
ShapeRequest.clipMinVoxel={ClipMin.X,ClipMin.Y,ClipMin.Z};
ShapeRequest.clipMaxVoxel={ClipMax.X,ClipMax.Y,ClipMax.Z};
ShapeRequest.workerLimit=static_cast<uint32>(std::clamp(WorkerCount,1,16));
ShapeRequest.detail=DetailVariant;ShapeRequest.includeFarWorld=bIncludeFarWorld;
ShapeRequest.roundTunnels=bRoundTunnels;
ShapeRequest.enhancedTunnelWalls=bEnhancedTunnelWalls;
FStandaloneStreamingGenerator Streamer(World,WorkerCount,bHashDensity,nullptr,MaxLevel,
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors,&Shape,bProfileColumnOverlap,
FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache);
Streamer.SetUniformBlockProofAuditEnabled(bAuditUniformBlockProofs);
if(bScreenSpaceGate||bReportLodWorkProfile)Streamer.SetMesherStageProfilingEnabled(true);
const auto StreamStart=Clock::now();
const FStandaloneStreamingUpdate EarlyUpdate=Streamer.StartWithDesired(ShapeFreeDesired);
if(EarlyUpdate.DesiredTiles!=static_cast<int32>(ShapeFreeDesired.Tiles.size()))
{
Streamer.Stop();
std::fprintf(stderr,"[StreamPipeline] expected %zu shape-free tiles but queued %d.\n",
ShapeFreeDesired.Tiles.size(),EarlyUpdate.DesiredTiles);return 7;
}
const auto ShapeBuildStart=Clock::now();
auto ShapeBuildFuture=std::async(std::launch::async,
[&Shape,&Scheduler,ShapeRequest,DiagnosticSlowShapeCompileMs]() mutable
{
if(DiagnosticSlowShapeCompileMs>0)
std::this_thread::sleep_for(std::chrono::milliseconds(DiagnosticSlowShapeCompileMs));
std::string Error;
const bool bCompiled=Shape.compile(ShapeRequest,Scheduler,Error);
return std::pair<bool,std::string>{bCompiled,std::move(Error)};
});
const auto [bShapeCompiled,BuildError]=ShapeBuildFuture.get();
if(!bShapeCompiled)
{
Streamer.Stop();
std::fprintf(stderr,"[StreamPipeline] full shape build failed: %s\n",BuildError.c_str());return 2;
}
const auto ShapeReady=Clock::now();
const double ShapeSeconds=std::chrono::duration<double>(ShapeReady-ShapeBuildStart).count();
if(bAuditTileShapeReadiness)
{
VoxelForgeCore::TileGenerationOptions CompiledOptions;
VoxelForgeCore::TileGenerationOptions NativeOnlyOptions;
CompiledOptions.hashDensity=true;
NativeOnlyOptions.hashDensity=true;
NativeOnlyOptions.diagnosticForceNativeOnly=true;
int32 Compared=0,Mismatched=0;
for(const FVoxelTileKey& Tile:ShapeFreeTiles)
{
if(!Neighborhood.contains(Tile))continue;
const VoxelForgeCore::TileKey CoreKey{{Tile.Coord.X,Tile.Coord.Y,Tile.Coord.Z},Tile.Level};
std::string CompiledError,NativeOnlyError;
const auto Compiled=Shape.generateTile(CoreKey,CompiledOptions,CompiledError);
const auto NativeOnly=Shape.generateTile(CoreKey,NativeOnlyOptions,NativeOnlyError);
++Compared;
const bool bSame=Compiled&&NativeOnly
&&Compiled->densityHash==NativeOnly->densityHash
&&Compiled->meshHash==NativeOnly->meshHash
&&Compiled->payloadHash==NativeOnly->payloadHash
&&Compiled->topologyHash==NativeOnly->topologyHash;
if(!bSame)
{
++Mismatched;
std::printf("[ShapeReadinessEquivalence] key=(%d,%d,%d,lod%d) match=0 compiled=%s native_only=%s density=%s mesh=%s\n",
Tile.Coord.X,Tile.Coord.Y,Tile.Coord.Z,Tile.Level,
Compiled?"ok":CompiledError.c_str(),NativeOnly?"ok":NativeOnlyError.c_str(),
Compiled&&NativeOnly&&Compiled->densityHash==NativeOnly->densityHash?"same":"different",
Compiled&&NativeOnly&&Compiled->meshHash==NativeOnly->meshHash?"same":"different");
}
}
std::printf("[ShapeReadinessEquivalence] compared_near_ring=%d mismatches=%d status=%s\n",
Compared,Mismatched,Mismatched==0?"PASS":"FAIL");
if(Mismatched!=0)
{
Streamer.Stop();
std::fprintf(stderr,"[StreamPipeline] native-only tile generation differs from compiled generation.\n");
return 9;
}
}
if(bAuditBasePageEquivalence)
{
std::vector<VoxelForgeCore::TileKey> AuditKeys;
AuditKeys.reserve(48);
const auto AddAuditCandidates=[&](int32 PreferredLOD)
{
const std::size_t ExistingCount=AuditKeys.size();
for(const FVoxelTileKey& Tile:Desired.Tiles)
{
if(Tile.Level!=PreferredLOD)continue;
const VoxelForgeCore::TileKey Key{{Tile.Coord.X,Tile.Coord.Y,Tile.Coord.Z},
Tile.Level};
if(Shape.isTileShapeFree(Key)||Shape.needsTileRegion(Key))continue;
if(std::find(AuditKeys.begin(),AuditKeys.end(),Key)==AuditKeys.end())
AuditKeys.push_back(Key);
if(AuditKeys.size()>=48||AuditKeys.size()-ExistingCount>=4)return;
}
};
for(int32 PreferredLOD:{1,0,2,3,4,5,6,7,8})
{
AddAuditCandidates(PreferredLOD);
if(AuditKeys.size()>=48)break;
}
if(AuditKeys.empty())
{
std::printf("[BasePageTileEquivalence] status=UNAVAILABLE reason=no_tile_reachable_through_both_routes\n");
Streamer.Stop();
return 9;
}
VoxelForgeCore::TileGenerationOptions HashOptions;
HashOptions.hashDensity=true;
HashOptions.profileMesherStages=true;
bool bFoundEquivalentTile=false;
std::size_t CandidatesTried=0;
VoxelForgeCore::TileKey EquivalentKey{};
for(const VoxelForgeCore::TileKey& AuditKey:AuditKeys)
{
++CandidatesTried;
std::string BaseError,PageError,CompileError;
const auto BaseGenerationStart=Clock::now();
const auto BaseTile=Shape.generateTile(AuditKey,HashOptions,BaseError);
const double BaseGenerationSeconds=std::chrono::duration<double>(
Clock::now()-BaseGenerationStart).count();
if(!BaseTile)
{
std::printf("[BasePageTileEquivalenceCandidate] key=(%d,%d,%d,lod%d) base=FAIL reason=%s\n",
AuditKey.coordinate.x,AuditKey.coordinate.y,AuditKey.coordinate.z,
AuditKey.lod,BaseError.c_str());
continue;
}
const std::array<VoxelForgeCore::TileKey,1> AuditTiles={AuditKey};
VoxelForgeCore::TileRegionCompileStats PageBuildStats;
std::size_t PageTileCount=0;
const auto PageCompileStart=Clock::now();
const bool bPageBuilt=Shape.compileTileRegion(AuditTiles,
static_cast<std::uint32_t>(std::clamp(WorkerCount,1,16)),Scheduler,
CompileError,&PageTileCount,&PageBuildStats,true,true);
const double PageCompileSeconds=std::chrono::duration<double>(
Clock::now()-PageCompileStart).count();
const auto PageGenerationStart=Clock::now();
const auto PageTile=bPageBuilt?Shape.generateTile(AuditKey,HashOptions,PageError)
:std::optional<VoxelForgeCore::TileMesh>{};
const double PageGenerationSeconds=std::chrono::duration<double>(
Clock::now()-PageGenerationStart).count();
const bool bDensitySame=PageTile&&BaseTile->densityHash==PageTile->densityHash;
const bool bMeshSame=PageTile&&BaseTile->meshHash==PageTile->meshHash;
const bool bPayloadSame=PageTile&&BaseTile->payloadHash==PageTile->payloadHash;
const bool bTopologySame=PageTile&&BaseTile->topologyHash==PageTile->topologyHash;
const bool bActualPage=PageTile&&PageTile->diagnostics.analyticRegionPageRoute;
const int32 Matches=static_cast<int32>(bDensitySame)+static_cast<int32>(bMeshSame)
+static_cast<int32>(bPayloadSame)+static_cast<int32>(bTopologySame);
const bool bSame=bActualPage&&Matches==4;
std::printf("[BasePageTileEquivalenceCandidate] key=(%d,%d,%d,lod%d) page_built=%d page_route_verified=%d page_keys=%zu page_nodes=%llu base_route=base-shape page_route=analytic-page base_generation_ms=%.3f page_compile_ms=%.3f page_generation_ms=%.3f density=%s mesh=%s payload=%s topology=%s base_density=0x%016llx page_density=0x%016llx status=%s error=%s\n",
AuditKey.coordinate.x,AuditKey.coordinate.y,AuditKey.coordinate.z,AuditKey.lod,
bPageBuilt?1:0,bActualPage?1:0,PageTileCount,
static_cast<unsigned long long>(PageBuildStats.latticeNodes),
BaseGenerationSeconds*1000.0,PageCompileSeconds*1000.0,
PageGenerationSeconds*1000.0,
bDensitySame?"same":"DIFF",bMeshSame?"same":"DIFF",
bPayloadSame?"same":"DIFF",bTopologySame?"same":"DIFF",
static_cast<unsigned long long>(BaseTile->densityHash),
static_cast<unsigned long long>(PageTile?PageTile->densityHash:0),
bSame?"PASS":"DIFF",bPageBuilt?PageError.c_str():CompileError.c_str());
if(bSame)
{
bFoundEquivalentTile=true;
EquivalentKey=AuditKey;
break;
}
}
std::printf("[BasePageTileEquivalence] candidates=%zu tried=%zu matching_key=(%d,%d,%d,lod%d) status=%s\n",
AuditKeys.size(),CandidatesTried,EquivalentKey.coordinate.x,
EquivalentKey.coordinate.y,EquivalentKey.coordinate.z,EquivalentKey.lod,
bFoundEquivalentTile?"PASS":"FAIL");
if(!bFoundEquivalentTile)
{
Streamer.Stop();
return 9;
}
}
const FStandaloneStreamingUpdate Update=Streamer.UpdateDesiredWithSet(Desired,{},0);
if(Update.DesiredTiles!=static_cast<int32>(Desired.Tiles.size()))
{
Streamer.Stop();
std::fprintf(stderr,"[StreamPipeline] expected %zu tiles after shape publication but queued %d.\n",
Desired.Tiles.size(),Update.DesiredTiles);return 7;
}
FSceneExportWriter SceneExport;
const bool bExportRequested=!MeshExportPath.empty();
bool bSceneExportPassing=true;
if(bExportRequested)
{
bSceneExportPassing=SceneExport.Open(MeshExportPath,World.CoreVoxelSizeMicrons,
CameraPositionVoxels,CameraYawDegrees,CameraPitchDegrees,
static_cast<std::uint32_t>(Update.DesiredTiles));
if(!bSceneExportPassing)
{
Streamer.Stop();
std::fprintf(stderr,"[MeshSceneExport] FAIL: could not open %s\n",
MeshExportPath.string().c_str());
return 8;
}
}
std::vector<FHashedTile> DensityRows,MeshRows,PayloadRows,TopologyRows;
int32 Finished=0,NeighborhoodFinished=0,FullDetailFinished=0;
int32 FullDetailExpected=0;
uint64 WholeNonEmptyMeshTiles=0,WholeTriangleCount=0;
struct FColumnXY
{
std::int64_t X=0,Y=0;
bool operator==(const FColumnXY&)const=default;
};
struct FColumnXYHash
{
std::size_t operator()(const FColumnXY& Value)const noexcept
{
const std::size_t X=std::hash<std::int64_t>{}(Value.X);
const std::size_t Y=std::hash<std::int64_t>{}(Value.Y);
return X^(Y+static_cast<std::size_t>(0x9E3779B9u)+(X<<6)+(X>>2));
}
};
struct FColumnEvaluationEvent
{
FColumnXY XY;
FVoxelTileKey Tile;
std::uint64_t WorkerHash=0;
std::uint64_t Sequence=0;
bool bOutsideTileFootprint=false;
};
std::array<std::vector<FColumnEvaluationEvent>,9> ColumnEventsByLOD;
std::array<std::uint64_t,9> ColumnEvaluationEventsByLOD{},UniqueColumnCoordinatesByLOD{};
std::array<std::uint64_t,9> ColumnDuplicateEventsByLOD{},SameWorkerEvictedRepeatsByLOD{};
std::array<std::uint64_t,9> DifferentWorkerRepeatsByLOD{},HaloNormalRepeatsByLOD{};
std::array<std::uint64_t,9> HaloNormalEvaluationsByLOD{},NeighbourDuplicateEventsByLOD{};
std::array<std::uint64_t,9> CrossWorkerNeighbourDuplicateEventsByLOD{};
std::array<std::uint64_t,9> ColumnCaptureTilesByLOD{},ExpectedTilesByLOD{};
std::array<bool,9> ColumnCaptureCompleteByLOD{};
ColumnCaptureCompleteByLOD.fill(true);
std::array<double,9> WorkerSecondsByLOD{};
std::array<double,9> BaseRouteWorkerSecondsByLOD{},PageRouteWorkerSecondsByLOD{};
std::array<double,9> ColumnEvalMsByLOD{},MarchingExclusiveMsByLOD{};
std::array<double,9> GroundRefineExclusiveMsByLOD{},CapRefineExclusiveMsByLOD{};
std::array<double,9> MeshAssemblyMsByLOD{};
std::array<double,9> SkirtMsByLOD{};
std::array<std::uint64_t,9> DensitySamplesByLOD{};
std::array<std::uint64_t,9> SkirtTrianglesByLOD{};
std::array<std::uint64_t,9> TileCountByLOD{};
std::array<std::uint64_t,9> BaseRouteTilesByLOD{},PageRouteTilesByLOD{};
for(const FVoxelTileKey& Tile:Desired.Tiles)
if(Tile.Level>=0&&Tile.Level<static_cast<int32>(ExpectedTilesByLOD.size()))
++ExpectedTilesByLOD[static_cast<std::size_t>(Tile.Level)];
const auto AreSameLODNeighbours=[](const FVoxelTileKey& A,const FVoxelTileKey& B)
{
if(A.Level!=B.Level)return false;
const std::int64_t DX=std::abs(static_cast<std::int64_t>(A.Coord.X)-B.Coord.X);
const std::int64_t DY=std::abs(static_cast<std::int64_t>(A.Coord.Y)-B.Coord.Y);
const std::int64_t DZ=std::abs(static_cast<std::int64_t>(A.Coord.Z)-B.Coord.Z);
return std::max({DX,DY,DZ})<=1&&(DX+DY+DZ)>0;
};
uint64 NativeSolidBoxes=0,NativeAirBoxes=0,NativeMixedBoxes=0,NativeUnknownBoxes=0;
uint64 NativeFieldCalls=0,NativeColumnHits=0,NativeColumnMisses=0,NativeMeshingSkips=0;
double NativeClassificationMilliseconds=0.0,NativeColumnComputeMilliseconds=0.0;
double NativeMeshingMilliseconds=0.0;
for(const FVoxelTileKey& Tile:Desired.Tiles)
if(Tile.Level<World.FullResClipLevels)++FullDetailExpected;
std::array<std::optional<FStandaloneCpuTile>,9> GateSampleTiles{};
std::array<double,9> GateSampleDistances{};
GateSampleDistances.fill(std::numeric_limits<double>::infinity());
Clock::time_point FirstTileReady{},NeighborhoodReady{},FullDetailReady{},WholeViewReady{};
bool bFirstTileReported=false,bNeighborhoodReported=false,bFullDetailReported=false;
uint64 BlockProofBlocksChecked=0,BlockProofSamplesChecked=0;
uint64 BlockProofFallbackBlocks=0,BlockProofUnknownBlocks=0,BlockProofViolations=0;
while(Finished<Update.DesiredTiles)
{
std::vector<FStandaloneCpuTile> Ready=Streamer.TakeReady();
if(Ready.empty()){std::this_thread::sleep_for(std::chrono::milliseconds(1));continue;}
for(const FStandaloneCpuTile& Tile:Ready)
{
if(Tile.bGenerationFailed)
{
Streamer.Stop();
std::fprintf(stderr,"[StreamPipeline] a tile failed generation; refusing a partial result.\n");
return 9;
}
if(bExportRequested)SceneExport.Add(Tile);
++Finished;
if(bReportMeshCoverage)
{
const uint64 TileTriangleCount=static_cast<uint64>(Tile.Indices.size()/3);
WholeTriangleCount+=TileTriangleCount;
if(TileTriangleCount>0)++WholeNonEmptyMeshTiles;
}
if(Tile.Key.Level>=0&&Tile.Key.Level<static_cast<int32>(TileCountByLOD.size()))
{
const std::size_t LOD=static_cast<std::size_t>(Tile.Key.Level);
++TileCountByLOD[LOD];
WorkerSecondsByLOD[LOD]+=Tile.WorkerSeconds;
if(bReportLodWorkProfile)
{
if(Tile.CoreDiagnostics.analyticRegionPageRoute)
{
++PageRouteTilesByLOD[LOD];
PageRouteWorkerSecondsByLOD[LOD]+=Tile.WorkerSeconds;
}
else
{
++BaseRouteTilesByLOD[LOD];
BaseRouteWorkerSecondsByLOD[LOD]+=Tile.WorkerSeconds;
}
}
ColumnEvalMsByLOD[LOD]+=Tile.CoreDiagnostics.columnEvaluationExclusiveMilliseconds;
MarchingExclusiveMsByLOD[LOD]+=Tile.CoreDiagnostics.marchingCubesExclusiveMilliseconds;
GroundRefineExclusiveMsByLOD[LOD]+=Tile.CoreDiagnostics.groundRefinementExclusiveMilliseconds;
CapRefineExclusiveMsByLOD[LOD]+=Tile.CoreDiagnostics.capRefinementExclusiveMilliseconds;
MeshAssemblyMsByLOD[LOD]+=Tile.CoreDiagnostics.meshAssemblyMilliseconds;
SkirtMsByLOD[LOD]+=Tile.CoreDiagnostics.marchingCubesSkirtMilliseconds;
if(bReportLodWorkProfile)DensitySamplesByLOD[LOD]+=Tile.DensitySamples;
SkirtTrianglesByLOD[LOD]+=Tile.SkirtTriangleCount;
if(bAuditUniformBlockProofs)
{
BlockProofBlocksChecked+=Tile.CoreDiagnostics.uniformBlockProofsChecked;
BlockProofSamplesChecked+=Tile.CoreDiagnostics.uniformBlockProofSamplesChecked;
BlockProofFallbackBlocks+=Tile.CoreDiagnostics.uniformBlockProofFallbackBlocks;
BlockProofUnknownBlocks+=Tile.CoreDiagnostics.uniformBlockProofUnknownBlocks;
BlockProofViolations+=Tile.CoreDiagnostics.uniformBlockProofViolations;
}
if(bScreenSpaceGate&&Assembly&&Tile.Key.Level>=3&&Tile.Key.Level<9
&&!Tile.Vertices.empty()
&&StandaloneScreenSpaceGate::TileHasNativeSurface(Tile.Key,*Assembly,
static_cast<double>(World.CoreVoxelSizeMicrons)/1000000.0))
{
const double Distance=StandaloneScreenSpaceGate::TileAabbDistanceMeters(
Tile.Key,CameraPositionVoxels,
static_cast<double>(World.CoreVoxelSizeMicrons)/1000000.0);
const bool bCloser=Distance<GateSampleDistances[LOD]-1e-9;
const bool bTied=std::abs(Distance-GateSampleDistances[LOD])<=1e-9;
const bool bKeyLess=!GateSampleTiles[LOD]
||TileKeyLessForHash(Tile.Key,GateSampleTiles[LOD]->Key);
if(bCloser||(bTied&&bKeyLess))
{GateSampleDistances[LOD]=Distance;GateSampleTiles[LOD]=Tile;}
}
}
if(bProfileColumnOverlap&&Tile.Key.Level>=0
&&Tile.Key.Level<static_cast<int32>(ColumnCaptureTilesByLOD.size()))
{
const std::size_t LOD=static_cast<std::size_t>(Tile.Key.Level);
++ColumnCaptureTilesByLOD[LOD];
ColumnCaptureCompleteByLOD[LOD]=ColumnCaptureCompleteByLOD[LOD]
&&Tile.CoreDiagnostics.columnCoordinateCaptureComplete;
auto& Events=ColumnEventsByLOD[LOD];
for(const auto& Coordinate:Tile.CoreDiagnostics.evaluatedColumnCoordinates)
{
Events.push_back({{Coordinate.x,Coordinate.y},Tile.Key,
Tile.CoreDiagnostics.columnEvaluationWorkerHash,Coordinate.sequence,
Coordinate.outsideTileFootprint});
}
}
WholeViewReady=std::max(WholeViewReady,Tile.CompletedAt);
if(FirstTileReady.time_since_epoch().count()==0||Tile.CompletedAt<FirstTileReady)
FirstTileReady=Tile.CompletedAt;
if(Neighborhood.contains(Tile.Key))
{
++NeighborhoodFinished;
NeighborhoodReady=std::max(NeighborhoodReady,Tile.CompletedAt);
}
if(Tile.Key.Level<World.FullResClipLevels)
{
++FullDetailFinished;
FullDetailReady=std::max(FullDetailReady,Tile.CompletedAt);
NativeSolidBoxes+=Tile.CoreDiagnostics.solidBoxes;
NativeAirBoxes+=Tile.CoreDiagnostics.airBoxes;
NativeMixedBoxes+=Tile.CoreDiagnostics.mixedBoxes;
NativeUnknownBoxes+=Tile.CoreDiagnostics.unknownBoxes;
NativeFieldCalls+=Tile.CoreDiagnostics.fieldCalls;
NativeColumnHits+=Tile.CoreDiagnostics.columnCacheHits;
NativeColumnMisses+=Tile.CoreDiagnostics.columnCacheMisses;
NativeMeshingSkips+=Tile.CoreDiagnostics.meshingSkipped?1u:0u;
NativeClassificationMilliseconds+=Tile.CoreDiagnostics.classificationMilliseconds;
NativeColumnComputeMilliseconds+=Tile.CoreDiagnostics.columnComputeMilliseconds;
NativeMeshingMilliseconds+=Tile.CoreDiagnostics.meshingMilliseconds;
}
if(bHashDensity)
{
if(Tile.bHasDensityHash)DensityRows.push_back({Tile.Key,Tile.DensityHash});
if(Tile.bHasMeshHash)MeshRows.push_back({Tile.Key,Tile.MeshHash});
if(Tile.bHasPayloadHash)PayloadRows.push_back({Tile.Key,Tile.PayloadHash});
if(Tile.bHasTopologyHash)TopologyRows.push_back({Tile.Key,Tile.TopologyHash});
}
}
if(!bFirstTileReported&&FirstTileReady.time_since_epoch().count()!=0)
{
std::printf("[PipelineMilestone] first_tile_ready=%.3fs\n",
std::chrono::duration<double>(FirstTileReady-ColdStart).count());
bFirstTileReported=true;
}
if(!bNeighborhoodReported&&NeighborhoodFinished==static_cast<int32>(Neighborhood.size()))
{
std::printf("[PipelineMilestone] neighborhood_ready=%.3fs generated=%d/27\n",
std::chrono::duration<double>(NeighborhoodReady-ColdStart).count(),NeighborhoodFinished);
bNeighborhoodReported=true;
}
if(!bFullDetailReported&&FullDetailFinished==FullDetailExpected)
{
std::printf("[PipelineMilestone] full_detail_ready=%.3fs generated=%d/%d\n",
std::chrono::duration<double>(FullDetailReady-ColdStart).count(),
FullDetailFinished,FullDetailExpected);
bFullDetailReported=true;
}
}
if(bHasCameraStep)
{
const auto MovingStart=Clock::now();
double RegionCompileWallSeconds=0.0;
double RegionCompileCpuSeconds=0.0;
std::size_t RegionCompileRequested=0,RegionCompileCovered=0;
VoxelForgeCore::TileRegionCompileStats RegionCompileStats;
if(bRegionCrossing)
{
std::unordered_set<FVoxelTileKey,FStandaloneTileKeyHash> InitialKeys;
InitialKeys.reserve(Desired.Tiles.size()*2+1);
for(const FVoxelTileKey& Key:Desired.Tiles)InitialKeys.insert(Key);
std::vector<VoxelForgeCore::TileKey> EnteringCoreTiles;
EnteringCoreTiles.reserve(MovedDesired.Tiles.size());
for(const FVoxelTileKey& Key:MovedDesired.Tiles)
{
if(InitialKeys.contains(Key))continue;
EnteringCoreTiles.push_back({{Key.Coord.X,Key.Coord.Y,Key.Coord.Z},Key.Level});
}
RegionCompileRequested=EnteringCoreTiles.size();
const auto RegionCompileStart=Clock::now();
const std::clock_t RegionCompileCpuStart=std::clock();
std::string RegionCompileError;
if(!Shape.compileTileRegion(EnteringCoreTiles,
static_cast<uint32>(std::clamp(WorkerCount,1,16)),Scheduler,
RegionCompileError,&RegionCompileCovered,&RegionCompileStats,true))
{
Streamer.Stop();
std::fprintf(stderr,"[MovingRegionCompile] failed: %s\n",RegionCompileError.c_str());
return 2;
}
RegionCompileWallSeconds=std::chrono::duration<double>(
Clock::now()-RegionCompileStart).count();
RegionCompileCpuSeconds=static_cast<double>(std::clock()-RegionCompileCpuStart)
/CLOCKS_PER_SEC;
const std::size_t ParallelWorkers=RegionCompileStats.regionsBuilt>1
?std::min<std::size_t>(8,RegionCompileStats.regionsBuilt):0;
std::printf("[MovingRegionCompile] distance_m=%.3f entering_tiles=%zu covered_tiles=%zu regions=%zu lattice_nodes=%llu wall_s=%.6f cpu_s=%.6f worker_threads=%zu mode=analytic-shape-pages\n",
std::sqrt(static_cast<double>(CameraStepVoxels.X)*CameraStepVoxels.X
+static_cast<double>(CameraStepVoxels.Y)*CameraStepVoxels.Y
+static_cast<double>(CameraStepVoxels.Z)*CameraStepVoxels.Z)
*static_cast<double>(World.CoreVoxelSizeMicrons)/1000000.0,
RegionCompileRequested,RegionCompileCovered,RegionCompileStats.regionsBuilt,
static_cast<unsigned long long>(RegionCompileStats.latticeNodes),
RegionCompileWallSeconds,RegionCompileCpuSeconds,ParallelWorkers);
}
const auto MovingTileStart=Clock::now();
const auto MovingUpdate=Streamer.UpdateDesiredWithSet(MovedDesired,Desired.Tiles,1);
std::array<std::uint64_t,9> MovingTilesByLOD{};
std::array<double,9> MovingWorkerSecondsByLOD{};
std::array<std::uint64_t,9> MovingSamplesByLOD{},MovingNativeFloatCallsByLOD{},
MovingNativeLatticeCallsByLOD{},MovingNativeOnlyTilesByLOD{},
MovingAnalyticPageTilesByLOD{},MovingNativeSurfaceTilesByLOD{};
std::array<std::uint64_t,9> MovingBaseRouteTilesByLOD{};
std::array<double,9> MovingBaseRouteWorkerSecondsByLOD{},
MovingPageRouteWorkerSecondsByLOD{};
std::array<double,9> MovingColumnComputeMsByLOD{},MovingClassificationMsByLOD{},
MovingMcExclusiveMsByLOD{},MovingMcDensityGridMsByLOD{},
MovingMcCellClassMsByLOD{},MovingMcGradientMsByLOD{},
MovingMcInterpolationMsByLOD{},MovingMcDedupMsByLOD{},MovingMcOtherMsByLOD{};
std::vector<FHashedTile> MovingDensityRows,MovingMeshRows,
MovingPayloadRows,MovingTopologyRows;
int32 MovingFinished=0;
while(MovingFinished<MovingUpdate.NewJobs)
{
std::vector<FStandaloneCpuTile> Ready=Streamer.TakeReady();
if(Ready.empty())
{std::this_thread::sleep_for(std::chrono::milliseconds(1));continue;}
for(const FStandaloneCpuTile& Tile:Ready)
{
++MovingFinished;
if(Tile.Key.Level>=0&&Tile.Key.Level<static_cast<int32>(MovingTilesByLOD.size()))
{
const std::size_t LOD=static_cast<std::size_t>(Tile.Key.Level);
++MovingTilesByLOD[LOD];
MovingWorkerSecondsByLOD[LOD]+=Tile.WorkerSeconds;
if(bReportLodWorkProfile)
{
const auto& D=Tile.CoreDiagnostics;
MovingSamplesByLOD[LOD]+=Tile.DensitySamples;
MovingNativeFloatCallsByLOD[LOD]+=D.floatFieldCalls;
MovingNativeLatticeCallsByLOD[LOD]+=D.latticeFieldCalls;
MovingNativeOnlyTilesByLOD[LOD]+=D.nativeOnlyRoute?1u:0u;
MovingAnalyticPageTilesByLOD[LOD]+=D.analyticRegionPageRoute?1u:0u;
MovingNativeSurfaceTilesByLOD[LOD]+=D.nativeSurfaceTile?1u:0u;
if(D.analyticRegionPageRoute)
MovingPageRouteWorkerSecondsByLOD[LOD]+=Tile.WorkerSeconds;
else
{
++MovingBaseRouteTilesByLOD[LOD];
MovingBaseRouteWorkerSecondsByLOD[LOD]+=Tile.WorkerSeconds;
}
MovingColumnComputeMsByLOD[LOD]+=D.columnEvaluationExclusiveMilliseconds;
MovingClassificationMsByLOD[LOD]+=D.classificationExclusiveMilliseconds;
MovingMcExclusiveMsByLOD[LOD]+=D.marchingCubesExclusiveMilliseconds;
MovingMcDensityGridMsByLOD[LOD]+=D.marchingCubesDensityGridMilliseconds;
MovingMcCellClassMsByLOD[LOD]+=D.marchingCubesCellClassificationMilliseconds;
MovingMcGradientMsByLOD[LOD]+=D.marchingCubesGradientNormalsMilliseconds;
MovingMcInterpolationMsByLOD[LOD]+=D.marchingCubesEdgeInterpolationMilliseconds;
MovingMcDedupMsByLOD[LOD]+=D.marchingCubesVertexDedupStreamMilliseconds;
MovingMcOtherMsByLOD[LOD]+=D.marchingCubesOtherMilliseconds;
}
}
if(bHashDensity)
{
if(Tile.bHasDensityHash)MovingDensityRows.push_back({Tile.Key,Tile.DensityHash});
if(Tile.bHasMeshHash)MovingMeshRows.push_back({Tile.Key,Tile.MeshHash});
if(Tile.bHasPayloadHash)MovingPayloadRows.push_back({Tile.Key,Tile.PayloadHash});
if(Tile.bHasTopologyHash)MovingTopologyRows.push_back({Tile.Key,Tile.TopologyHash});
}
}
}
const double MovingTileWallSeconds=std::chrono::duration<double>(
Clock::now()-MovingTileStart).count();
const double MovingWallSeconds=std::chrono::duration<double>(Clock::now()-MovingStart).count();
const double StepDistanceMeters=std::sqrt(
static_cast<double>(CameraStepVoxels.X)*CameraStepVoxels.X
+static_cast<double>(CameraStepVoxels.Y)*CameraStepVoxels.Y
+static_cast<double>(CameraStepVoxels.Z)*CameraStepVoxels.Z)
*static_cast<double>(World.CoreVoxelSizeMicrons)/1000000.0;
std::printf("[MovingCameraStep] step_voxels=(%.3f,%.3f,%.3f) distance_m=%.3f desired_tiles=%zu new_tiles=%d completed=%d wall_s=%.6f shared_column_cache=%s\n",
CameraStepVoxels.X,CameraStepVoxels.Y,CameraStepVoxels.Z,StepDistanceMeters,
MovedDesired.Tiles.size(),MovingUpdate.NewJobs,MovingFinished,MovingWallSeconds,
bUseSharedColumnCache?"on":"off");
std::printf("[MovingTileWork] tiles=%d wall_s=%.6f worker_sum_s=%.6f region_compile_wall_s=%.6f crossing_total_s=%.6f\n",
MovingFinished,MovingTileWallSeconds,
std::accumulate(MovingWorkerSecondsByLOD.begin(),MovingWorkerSecondsByLOD.end(),0.0),
RegionCompileWallSeconds,MovingWallSeconds);
if(bHashDensity)
std::printf("[MovingTileHash] tiles=%d rows=(%zu,%zu,%zu,%zu) density=0x%016llx mesh=0x%016llx payload=0x%016llx topology=0x%016llx status=%s\n",
MovingFinished,MovingDensityRows.size(),MovingMeshRows.size(),
MovingPayloadRows.size(),MovingTopologyRows.size(),
static_cast<unsigned long long>(HashTileRows(MovingDensityRows)),
static_cast<unsigned long long>(HashTileRows(MovingMeshRows)),
static_cast<unsigned long long>(HashTileRows(MovingPayloadRows)),
static_cast<unsigned long long>(HashTileRows(MovingTopologyRows)),
MovingDensityRows.size()==static_cast<std::size_t>(MovingFinished)
&&MovingMeshRows.size()==static_cast<std::size_t>(MovingFinished)
&&MovingPayloadRows.size()==static_cast<std::size_t>(MovingFinished)
&&MovingTopologyRows.size()==static_cast<std::size_t>(MovingFinished)
?"PASS":"FAIL");
for(std::size_t LOD=0;LOD<MovingTilesByLOD.size();++LOD)
if(MovingTilesByLOD[LOD]!=0)
{
std::printf("[MovingCameraStepLOD] lod=%zu tiles=%llu worker_sum_s=%.6f\n",
LOD,static_cast<unsigned long long>(MovingTilesByLOD[LOD]),
MovingWorkerSecondsByLOD[LOD]);
if(bReportLodWorkProfile)
std::printf("[MovingCameraStepLODProfile] lod=%zu density_samples=%llu samples_per_tile=%.1f native_float_calls=%llu native_lattice_calls=%llu native_surface_tiles=%llu native_only_tiles=%llu analytic_page_tiles=%llu base_route_tiles=%llu base_route_avg_ms=%.3f page_route_avg_ms=%.3f column_compute_ms=%.3f classification_ms=%.3f mc_exclusive_ms=%.3f mc_density_grid_ms=%.3f mc_cell_class_ms=%.3f mc_gradient_ms=%.3f mc_interpolation_ms=%.3f mc_dedup_ms=%.3f mc_other_ms=%.3f\n",
LOD,static_cast<unsigned long long>(MovingSamplesByLOD[LOD]),
static_cast<double>(MovingSamplesByLOD[LOD])/MovingTilesByLOD[LOD],
static_cast<unsigned long long>(MovingNativeFloatCallsByLOD[LOD]),
static_cast<unsigned long long>(MovingNativeLatticeCallsByLOD[LOD]),
static_cast<unsigned long long>(MovingNativeSurfaceTilesByLOD[LOD]),
static_cast<unsigned long long>(MovingNativeOnlyTilesByLOD[LOD]),
static_cast<unsigned long long>(MovingAnalyticPageTilesByLOD[LOD]),
static_cast<unsigned long long>(MovingBaseRouteTilesByLOD[LOD]),
MovingBaseRouteTilesByLOD[LOD]
?MovingBaseRouteWorkerSecondsByLOD[LOD]*1000.0
/MovingBaseRouteTilesByLOD[LOD]:0.0,
MovingAnalyticPageTilesByLOD[LOD]
?MovingPageRouteWorkerSecondsByLOD[LOD]*1000.0
/MovingAnalyticPageTilesByLOD[LOD]:0.0,
MovingColumnComputeMsByLOD[LOD],MovingClassificationMsByLOD[LOD],
MovingMcExclusiveMsByLOD[LOD],MovingMcDensityGridMsByLOD[LOD],
MovingMcCellClassMsByLOD[LOD],MovingMcGradientMsByLOD[LOD],
MovingMcInterpolationMsByLOD[LOD],MovingMcDedupMsByLOD[LOD],
MovingMcOtherMsByLOD[LOD]);
}
}
Streamer.Stop();
bool bScreenSpaceGatePassing=true;
if(bScreenSpaceGate)
{
if(Assembly)
bScreenSpaceGatePassing=StandaloneScreenSpaceGate::Run(GateSampleTiles,
Streamer,*Assembly,ClosestDisplayDistances,World.CoreVoxelSizeMicrons,
World.MaxClipLevel,PixelBudgetsSubvoxels,CameraYawDegrees,
CameraPitchDegrees);
else
{std::fprintf(stderr,"[ScreenSpaceGateResult] no-world-assembly status=REJECT\n");bScreenSpaceGatePassing=false;}
}
if(bExportRequested)
{
bSceneExportPassing=SceneExport.Close(static_cast<std::uint32_t>(Update.DesiredTiles));
std::error_code FileSizeError;
const std::uintmax_t ExportBytes=std::filesystem::file_size(MeshExportPath,FileSizeError);
std::printf("[MeshSceneExport] tiles=%u expected=%d bytes=%llu status=%s path=%s\n",
SceneExport.TileCount,Update.DesiredTiles,
static_cast<unsigned long long>(FileSizeError?0:ExportBytes),
bSceneExportPassing?"PASS":"FAIL",MeshExportPath.string().c_str());
bSceneExportPassing=bSceneExportPassing&&!FileSizeError;
}
if(bProfileColumnOverlap)
{
struct FColumnPriorUse
{
FVoxelTileKey Tile;
std::uint64_t WorkerHash=0;
};
for(std::size_t LOD=0;LOD<ColumnEventsByLOD.size();++LOD)
{
auto& Events=ColumnEventsByLOD[LOD];
std::sort(Events.begin(),Events.end(),[](const FColumnEvaluationEvent& A,
const FColumnEvaluationEvent& B){return A.Sequence<B.Sequence;});
std::unordered_map<FColumnXY,std::vector<FColumnPriorUse>,FColumnXYHash> UsesByXY;
UsesByXY.reserve(Events.size());
for(const FColumnEvaluationEvent& Event:Events)
{
++ColumnEvaluationEventsByLOD[LOD];
if(Event.bOutsideTileFootprint)++HaloNormalEvaluationsByLOD[LOD];
auto [Found,Inserted]=UsesByXY.try_emplace(Event.XY);
auto& PriorUses=Found->second;
if(Inserted)
{
++UniqueColumnCoordinatesByLOD[LOD];
PriorUses.push_back({Event.Tile,Event.WorkerHash});
continue;
}
++ColumnDuplicateEventsByLOD[LOD];
if(Event.bOutsideTileFootprint)
++HaloNormalRepeatsByLOD[LOD];
else
{
const bool bSameWorkerSeen=std::any_of(PriorUses.begin(),PriorUses.end(),
[&](const FColumnPriorUse& Prior)
{return Prior.WorkerHash==Event.WorkerHash;});
if(bSameWorkerSeen)++SameWorkerEvictedRepeatsByLOD[LOD];
else ++DifferentWorkerRepeatsByLOD[LOD];
}
bool bHasNeighbourDuplicate=false,bHasCrossWorkerNeighbourDuplicate=false;
for(const FColumnPriorUse& Prior:PriorUses)
{
if(!AreSameLODNeighbours(Prior.Tile,Event.Tile))continue;
bHasNeighbourDuplicate=true;
bHasCrossWorkerNeighbourDuplicate=bHasCrossWorkerNeighbourDuplicate
||Prior.WorkerHash!=Event.WorkerHash;
}
if(bHasNeighbourDuplicate)++NeighbourDuplicateEventsByLOD[LOD];
if(bHasCrossWorkerNeighbourDuplicate)
++CrossWorkerNeighbourDuplicateEventsByLOD[LOD];
PriorUses.push_back({Event.Tile,Event.WorkerHash});
}
}
}
bool bColumnOverlapCapturePassing=true;
if(bProfileColumnOverlap)
{
for(std::size_t LOD=0;LOD<ColumnCaptureTilesByLOD.size();++LOD)
{
const bool bLODComplete=ColumnCaptureCompleteByLOD[LOD]
&&ColumnCaptureTilesByLOD[LOD]==ExpectedTilesByLOD[LOD];
bColumnOverlapCapturePassing=bColumnOverlapCapturePassing&&bLODComplete;
std::printf("[WorldAssemblerLODColumnOverlap] lod=%zu tiles=%llu expected_tiles=%llu total_column_evaluations=%llu unique_xy_columns=%llu repeated_column_evaluations=%llu same_worker_evicted_repeats=%llu different_worker_repeats=%llu halo_normal_repeats=%llu halo_normal_evaluations=%llu neighbouring_duplicate_evaluations=%llu cross_worker_neighbouring_duplicates=%llu capture_complete=%d\n",
LOD,static_cast<unsigned long long>(ColumnCaptureTilesByLOD[LOD]),
static_cast<unsigned long long>(ExpectedTilesByLOD[LOD]),
static_cast<unsigned long long>(ColumnEvaluationEventsByLOD[LOD]),
static_cast<unsigned long long>(UniqueColumnCoordinatesByLOD[LOD]),
static_cast<unsigned long long>(ColumnDuplicateEventsByLOD[LOD]),
static_cast<unsigned long long>(SameWorkerEvictedRepeatsByLOD[LOD]),
static_cast<unsigned long long>(DifferentWorkerRepeatsByLOD[LOD]),
static_cast<unsigned long long>(HaloNormalRepeatsByLOD[LOD]),
static_cast<unsigned long long>(HaloNormalEvaluationsByLOD[LOD]),
static_cast<unsigned long long>(NeighbourDuplicateEventsByLOD[LOD]),
static_cast<unsigned long long>(CrossWorkerNeighbourDuplicateEventsByLOD[LOD]),
bLODComplete?1:0);
}
}
for(std::size_t LOD=0;LOD<TileCountByLOD.size();++LOD)
{
std::printf("[WorldAssemblerLODWorker] lod=%zu tiles=%llu worker_sum_s=%.6f column_eval_exclusive_ms=%.3f mc_exclusive_ms=%.3f ground_refine_exclusive_ms=%.3f cap_refine_exclusive_ms=%.3f skirt_ms=%.3f skirt_triangles=%llu mesh_assembly_ms=%.3f\n",
LOD,static_cast<unsigned long long>(TileCountByLOD[LOD]),WorkerSecondsByLOD[LOD],
ColumnEvalMsByLOD[LOD],MarchingExclusiveMsByLOD[LOD],
GroundRefineExclusiveMsByLOD[LOD],CapRefineExclusiveMsByLOD[LOD],
SkirtMsByLOD[LOD],static_cast<unsigned long long>(SkirtTrianglesByLOD[LOD]),
MeshAssemblyMsByLOD[LOD]);
if(bReportLodWorkProfile)
{
const double SamplesPerTile=TileCountByLOD[LOD]
?static_cast<double>(DensitySamplesByLOD[LOD])/TileCountByLOD[LOD]:0.0;
std::printf("[WorldAssemblerLODSamples] lod=%zu samples=%llu samples_per_tile=%.1f\n",
LOD,static_cast<unsigned long long>(DensitySamplesByLOD[LOD]),SamplesPerTile);
std::printf("[WorldAssemblerLODRouteProfile] lod=%zu base_shape_tiles=%llu base_shape_avg_ms=%.3f analytic_page_tiles=%llu analytic_page_avg_ms=%.3f\n",
LOD,static_cast<unsigned long long>(BaseRouteTilesByLOD[LOD]),
BaseRouteTilesByLOD[LOD]
?BaseRouteWorkerSecondsByLOD[LOD]*1000.0/BaseRouteTilesByLOD[LOD]:0.0,
static_cast<unsigned long long>(PageRouteTilesByLOD[LOD]),
PageRouteTilesByLOD[LOD]
?PageRouteWorkerSecondsByLOD[LOD]*1000.0/PageRouteTilesByLOD[LOD]:0.0);
}
}
const double NeighborhoodSeconds=std::chrono::duration<double>(NeighborhoodReady-ColdStart).count();
const double NeighborhoodMeshSeconds=std::max(0.0,
std::chrono::duration<double>(NeighborhoodReady-ShapeReady).count());
const double FullDetailSeconds=std::chrono::duration<double>(FullDetailReady-ColdStart).count();
const double WholeViewSeconds=std::chrono::duration<double>(WholeViewReady-ColdStart).count();
const double StreamSeconds=std::chrono::duration<double>(WholeViewReady-StreamStart).count();
const double FirstTileSeconds=std::chrono::duration<double>(FirstTileReady-ColdStart).count();
std::printf("[StreamPipeline] workers=%d shape=%.3fs first_tile_ready=%.3fs neighborhood_mesh=%.3fs neighborhood_ready=%.3fs full_detail_ready=%.3fs whole_view_ready=%.3fs neighborhood_tiles=%d/27 full_detail_tiles=%d/%d whole_tiles=%d/%zu stream_wall=%.3fs cache_hits=0 full_payload=1\n",
WorkerCount,ShapeSeconds,FirstTileSeconds,NeighborhoodMeshSeconds,NeighborhoodSeconds,
FullDetailSeconds,WholeViewSeconds,NeighborhoodFinished,
FullDetailFinished,FullDetailExpected,Finished,Desired.Tiles.size(),StreamSeconds);
if(bReportMeshCoverage)
std::printf("[PipelineMeshCoverage] tiles=%d nonempty_mesh_tiles=%llu indexed_triangles=%llu\n",
Finished,static_cast<unsigned long long>(WholeNonEmptyMeshTiles),
static_cast<unsigned long long>(WholeTriangleCount));
std::printf("[WorldAssemblerTilePipeline] full_detail_tiles=%d solid_boxes=%llu air_boxes=%llu mixed_boxes=%llu unknown_boxes=%llu whole_tile_skips=%llu field_calls=%llu column_cache_hits=%llu column_cache_misses=%llu column_compute_worker_ms=%.3f classification_worker_ms=%.3f meshing_worker_ms=%.3f\n",
FullDetailExpected,static_cast<unsigned long long>(NativeSolidBoxes),
static_cast<unsigned long long>(NativeAirBoxes),
static_cast<unsigned long long>(NativeMixedBoxes),
static_cast<unsigned long long>(NativeUnknownBoxes),
static_cast<unsigned long long>(NativeMeshingSkips),
static_cast<unsigned long long>(NativeFieldCalls),
static_cast<unsigned long long>(NativeColumnHits),
static_cast<unsigned long long>(NativeColumnMisses),
NativeColumnComputeMilliseconds,NativeClassificationMilliseconds,
NativeMeshingMilliseconds);
if(bAuditUniformBlockProofs)
std::printf("[UniformBlockProofAudit] checked_blocks=%llu checked_samples=%llu fallback_blocks=%llu unknown_blocks=%llu violations=%llu\n",
static_cast<unsigned long long>(BlockProofBlocksChecked),
static_cast<unsigned long long>(BlockProofSamplesChecked),
static_cast<unsigned long long>(BlockProofFallbackBlocks),
static_cast<unsigned long long>(BlockProofUnknownBlocks),
static_cast<unsigned long long>(BlockProofViolations));
bool bHashesPassing=true;
if(bHashDensity)
{
bHashesPassing=DensityRows.size()==Desired.Tiles.size()
&&MeshRows.size()==Desired.Tiles.size()
&&PayloadRows.size()==Desired.Tiles.size()
&&TopologyRows.size()==Desired.Tiles.size();
std::vector<FHashedTile> FullDensity,FullMesh,FullPayload,FullTopology;
for(const FHashedTile& Row:DensityRows)if(Row.Key.Level<World.FullResClipLevels)FullDensity.push_back(Row);
for(const FHashedTile& Row:MeshRows)if(Row.Key.Level<World.FullResClipLevels)FullMesh.push_back(Row);
for(const FHashedTile& Row:PayloadRows)if(Row.Key.Level<World.FullResClipLevels)FullPayload.push_back(Row);
for(const FHashedTile& Row:TopologyRows)if(Row.Key.Level<World.FullResClipLevels)FullTopology.push_back(Row);
std::printf("[PipelineFullDetailHash] tiles=%d density=0x%016llx mesh=0x%016llx payload=0x%016llx topology=0x%016llx\n",
FullDetailExpected,static_cast<unsigned long long>(HashTileRows(FullDensity)),
static_cast<unsigned long long>(HashTileRows(FullMesh)),
static_cast<unsigned long long>(HashTileRows(FullPayload)),
static_cast<unsigned long long>(HashTileRows(FullTopology)));
std::printf("[PipelineWholeViewHash] tiles=%zu density=0x%016llx mesh=0x%016llx payload=0x%016llx topology=0x%016llx\n",
Desired.Tiles.size(),static_cast<unsigned long long>(HashTileRows(DensityRows)),
static_cast<unsigned long long>(HashTileRows(MeshRows)),
static_cast<unsigned long long>(HashTileRows(PayloadRows)),
static_cast<unsigned long long>(HashTileRows(TopologyRows)));
for(std::size_t LOD=0;LOD<TileCountByLOD.size();++LOD)
{
const auto FilterLOD=[LOD](const std::vector<FHashedTile>& Rows)
{
std::vector<FHashedTile> Selected;
for(const FHashedTile& Row:Rows)
if(Row.Key.Level==static_cast<int32>(LOD))Selected.push_back(Row);
return Selected;
};
auto LodDensity=FilterLOD(DensityRows),LodMesh=FilterLOD(MeshRows);
auto LodPayload=FilterLOD(PayloadRows),LodTopology=FilterLOD(TopologyRows);
std::printf("[PipelineLODHash] lod=%zu tiles=%llu density=0x%016llx mesh=0x%016llx payload=0x%016llx topology=0x%016llx\n",
LOD,static_cast<unsigned long long>(TileCountByLOD[LOD]),
static_cast<unsigned long long>(HashTileRows(LodDensity)),
static_cast<unsigned long long>(HashTileRows(LodMesh)),
static_cast<unsigned long long>(HashTileRows(LodPayload)),
static_cast<unsigned long long>(HashTileRows(LodTopology)));
}
const std::array<VoxelForgeCore::Vec3d,9> QueryPoints={{
{CameraPositionVoxels.X,CameraPositionVoxels.Y,CameraPositionVoxels.Z},
{CameraPositionVoxels.X+8.0,CameraPositionVoxels.Y,CameraPositionVoxels.Z},
{CameraPositionVoxels.X-8.0,CameraPositionVoxels.Y,CameraPositionVoxels.Z},
{CameraPositionVoxels.X,CameraPositionVoxels.Y+8.0,CameraPositionVoxels.Z},
{CameraPositionVoxels.X,CameraPositionVoxels.Y-8.0,CameraPositionVoxels.Z},
{CameraPositionVoxels.X,CameraPositionVoxels.Y,CameraPositionVoxels.Z+8.0},
{CameraPositionVoxels.X,CameraPositionVoxels.Y,CameraPositionVoxels.Z-8.0},
{CameraPositionVoxels.X+64.0,CameraPositionVoxels.Y-32.0,CameraPositionVoxels.Z+16.0},
{CameraPositionVoxels.X-128.0,CameraPositionVoxels.Y+96.0,CameraPositionVoxels.Z-64.0}}};
uint64 ExactDensityHash=14695981039346656037ull;
uint64 ExactUnclampedHash=14695981039346656037ull;
uint64 ExactDetailHash=14695981039346656037ull;
uint64 ExactCollisionHash=14695981039346656037ull;
for(const auto& Point:QueryPoints)
{
const float Density=Shape.densityAt(Point);
const float Unclamped=Shape.unclampedDensityAt(Point);
const auto Detail=Shape.detailSampleAt(Point);
HashWord(ExactDensityHash,std::bit_cast<uint32>(Density),4);
HashWord(ExactUnclampedHash,std::bit_cast<uint32>(Unclamped),4);
HashWord(ExactDetailHash,std::bit_cast<uint32>(Detail.displacement),4);
HashWord(ExactDetailHash,std::bit_cast<uint32>(Detail.caveInfluence),4);
HashWord(ExactDetailHash,std::bit_cast<uint32>(Detail.floorInfluence),4);
HashWord(ExactCollisionHash,Shape.isSolidAt(Point)?1u:0u,1);
}
std::printf("[PipelineExactQueryHash] probes=%zu density=0x%016llx unclamped=0x%016llx detail=0x%016llx collision=0x%016llx path=direct-Core-queries\n",
QueryPoints.size(),static_cast<unsigned long long>(ExactDensityHash),
static_cast<unsigned long long>(ExactUnclampedHash),
static_cast<unsigned long long>(ExactDetailHash),
static_cast<unsigned long long>(ExactCollisionHash));
}
return NeighborhoodFinished==static_cast<int32>(Neighborhood.size())
&&FullDetailFinished==FullDetailExpected&&Finished==Update.DesiredTiles&&bHashesPassing
&&(!bProfileColumnOverlap||bColumnOverlapCapturePassing)
&&(!bExportRequested||bSceneExportPassing)&&bScreenSpaceGatePassing
&&(!bAuditUniformBlockProofs||BlockProofViolations==0)?0:6;
}
} // namespace StandaloneStreamBenchmark
Reference in new issue View Git Blame Copy Permalink