Fr0zka/VoxelForge
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip(L16, side branch): disk terrain cache design + headless Linux prototype

Browse Source 
Exact lossless zlib tile records (keyed on world/season/recipe/plan/evaluator/options/tile),
pristine cache plus ordered edits, server never trusts client caches, corruption/stale rejection,
atomic writes, 2 GB cap. Loaded tiles match regenerated hashes (L1 view, 24 m crossing).
NOT for assembler/world yet: the shared StreamBenchmarkPipeline.h includes Linux-only headers
(unistd/fcntl/zlib), which would break the Windows viewer build.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CSFYgsdW8ejZSyiMf4qRSX
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed 2026-10-20 04:19:56 +02:00
1 parent a6cd7cf339
commit b5ed321b9e
7 files changed
+1664 -13
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Standalone/Source/StreamBenchmarkPipeline.h
+112 -7
View File
@@ -3,6 +3,7 @@
#include "OwnerWorldLoader.h"
#include "StreamingWorld.h"
#include "LinuxScreenSpaceGate.h"
#include "../Research/WorldAssemblerM3/Linux16/TileCachePrototype.h"
#include <algorithm>
#include <array>
@@ -253,7 +254,9 @@ namespace StandaloneStreamBenchmark
bool bAuditUniformBlockProofs=false,bool bReportLodWorkProfile=false,
bool bRegionCrossing=false,
bool bAuditTileShapeReadiness=false,int DiagnosticSlowShapeCompileMs=0,
bool bAuditBasePageEquivalence=false,bool bAuditRouteParity=false)
bool bAuditBasePageEquivalence=false,bool bAuditRouteParity=false,
const std::filesystem::path& TileCachePrototypeRoot={},
const std::filesystem::path& OwnerSnapshotPath={})
{
using Clock=std::chrono::steady_clock;
FStandaloneOwnerWorld& World=*OwnerWorld;
@@ -274,6 +277,42 @@ namespace StandaloneStreamBenchmark
const bool bRoundTunnels=World.Generator->bStandaloneResearchRoundTunnels;
const bool bEnhancedTunnelWalls=World.Generator->bStandaloneResearchEnhancedWalls;
VoxelForgeCore::World Shape(CoreSnapshot);
std::unique_ptr<StandaloneTileCachePrototype::TileDiskCache> TileCache;
FStandaloneCoreTileRoute CoreTileRoute;
VoxelForgeCore::TileGenerationOptions CacheOptions;
if(!TileCachePrototypeRoot.empty())
{
const auto RepoRoot=std::filesystem::current_path().lexically_normal();
const auto AllowedRoot=(RepoRoot/"Saved/Linux/cache").lexically_normal();
const auto RequestedRoot=std::filesystem::absolute(TileCachePrototypeRoot).lexically_normal();
const std::string AllowedString=AllowedRoot.string()+std::string(1,std::filesystem::path::preferred_separator);
if(RequestedRoot.string().rfind(AllowedString,0)!=0||OwnerSnapshotPath.empty())
{
std::fprintf(stderr,"[TileCachePrototype] path rejected: cache writes must stay under Saved/Linux/cache and owner snapshot identity must be provided.\n");
return 2;
}
if(ClosestDisplayDistances[0]==0.0)
ClosestDisplayDistances=StandaloneScreenSpaceGate::ClosestDisplayedDistancesMeters(
World,CameraPositionVoxels,World.CoreVoxelSizeMicrons);
const auto Identity=StandaloneTileCachePrototype::MakeIdentity(
World,CoreSnapshot,OwnerSnapshotPath);
if(Identity.ownerSnapshotHash==0||!Assembly)
{
std::fprintf(stderr,"[TileCachePrototype] missing owner snapshot or world assembly identity.\n");
return 2;
}
TileCache=std::make_unique<StandaloneTileCachePrototype::TileDiskCache>(
RequestedRoot,Identity,ClosestDisplayDistances);
CacheOptions.hashDensity=true;
StandaloneTileCachePrototype::TileDiskCache* Cache=TileCache.get();
CoreTileRoute=[Cache](const VoxelForgeCore::World& TileWorld,
const VoxelForgeCore::TileKey& Key,
const VoxelForgeCore::TileGenerationOptions& Options,bool& bHit,
std::string& Error)
{return Cache->GenerateOrLoad(TileWorld,Key,Options,bHit,Error);};
std::printf("[TileCachePrototype] enabled root=%s owner_snapshot=%s mode=worker_generate_store_read_or_warm_hit codec=zlib-6\n",
RequestedRoot.string().c_str(),OwnerSnapshotPath.string().c_str());
}
FStandaloneDesiredTileSet Desired=StandaloneWorldMesher::BuildDesiredSet(World,CameraPositionVoxels);
const auto CoreTiles=Shape.desiredTiles(
{CameraPositionVoxels.X,CameraPositionVoxels.Y,CameraPositionVoxels.Z});
@@ -411,7 +450,7 @@ namespace StandaloneStreamBenchmark
ShapeRequest.enhancedTunnelWalls=bEnhancedTunnelWalls;
FStandaloneStreamingGenerator Streamer(World,WorkerCount,bHashDensity,nullptr,MaxLevel,
bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors,&Shape,bProfileColumnOverlap,
FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache);
FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache,std::move(CoreTileRoute));
Streamer.SetUniformBlockProofAuditEnabled(bAuditUniformBlockProofs);
if(bScreenSpaceGate||bReportLodWorkProfile)Streamer.SetMesherStageProfilingEnabled(true);
const auto StreamStart=Clock::now();
@@ -657,6 +696,7 @@ namespace StandaloneStreamBenchmark
VoxelForgeCore::TileGenerationOptions PageOptions=CaptureOptions;
PageOptions.auditUniformBlockProofs=true;
int32 CachePairs=0,CacheMismatches=0,EditCachePairs=0,EditCacheMismatches=0;
int32 BasePageMismatches=0,PageRoutesVerified=0;
std::uint64_t PageProofBlocks=0,PageProofSamples=0,PageProofFallbacks=0,
PageProofUnknown=0,PageProofViolations=0;
@@ -716,6 +756,28 @@ namespace StandaloneStreamBenchmark
PageProofUnknown+=Page->diagnostics.uniformBlockProofUnknownBlocks;
PageProofViolations+=Page->diagnostics.uniformBlockProofViolations;
}
if(TileCache)
{
VoxelForgeCore::TileMesh Cached;
std::string CacheError;
bool bCacheReady=TileCache->LoadExpected(Row.Candidate.Key,CacheOptions,Cached,CacheError);
if(!bCacheReady&&Row.Base)
bCacheReady=TileCache->StoreAndLoadExplicit(*Row.Base,CacheOptions,
TileCache->identity(),Cached,CacheError);
const bool bCacheSame=bCacheReady&&Row.Base
&&SameTileOutput(*Row.Base,Cached)
&&StandaloneTileCachePrototype::SameExactMesh(*Row.Base,Cached);
++CachePairs;
if(!bCacheSame)++CacheMismatches;
std::printf("[RouteParityPair] route=base/disk-cache key=(%d,%d,%d,lod%d) same_density=%d same_mesh=%d same_payload=%d same_topology=%d exact_arrays=%d status=%s error=%s\n",
Row.Candidate.Key.coordinate.x,Row.Candidate.Key.coordinate.y,
Row.Candidate.Key.coordinate.z,Row.Candidate.Key.lod,
Row.Base&&Cached.densityHash==Row.Base->densityHash?1:0,
Row.Base&&Cached.meshHash==Row.Base->meshHash?1:0,
Row.Base&&Cached.payloadHash==Row.Base->payloadHash?1:0,
Row.Base&&Cached.topologyHash==Row.Base->topologyHash?1:0,
bCacheSame?1:0,bCacheSame?"PASS":"FAIL",CacheError.c_str());
}
std::printf("[RouteParityPair] route=base/page key=(%d,%d,%d,lod%d) page_requested=%d page_verified=%d same_density=%d same_mesh=%d same_payload=%d same_topology=%d samples=%llu max_abs_density_delta=%.9g sign_mismatches=%llu edge_crossing_mismatches=%llu shared_surface_edges=%llu max_surface_delta_vox=%.9g exact_query_mismatches=%llu base_native_float=%llu page_native_float=%llu first_density=(%.9g,%.9g) status=%s error=%s\n",
Row.Candidate.Key.coordinate.x,Row.Candidate.Key.coordinate.y,
Row.Candidate.Key.coordinate.z,Row.Candidate.Key.lod,
@@ -899,6 +961,27 @@ namespace StandaloneStreamBenchmark
&&EditedPage->diagnostics.analyticRegionPageRoute;
const bool bEditSame=EditedBase&&EditedPage&&bEditPageVerified
&&SameTileOutput(*EditedBase,*EditedPage);
bool bEditCacheSame=false;
if(TileCache&&EditedBase&&Applied)
{
VoxelForgeCore::TileMesh CachedEdited;
std::string CacheError;
const auto EditedIdentity=TileCache->WithEdit(Transaction.sequence,Applied->digest);
bEditCacheSame=TileCache->StoreAndLoadExplicit(*EditedBase,CacheOptions,
EditedIdentity,CachedEdited,CacheError)
&&SameTileOutput(*EditedBase,CachedEdited);
++EditCachePairs;
if(!bEditCacheSame)++EditCacheMismatches;
std::printf("[RouteParityPair] route=edited-base/disk-cache key=(%d,%d,%d,lod%d) edit_sequence=%llu edit_digest=0x%016llx same_density=%d same_mesh=%d same_payload=%d same_topology=%d exact_arrays=%d status=%s error=%s\n",
Key.coordinate.x,Key.coordinate.y,Key.coordinate.z,Key.lod,
static_cast<unsigned long long>(Transaction.sequence),
static_cast<unsigned long long>(Applied->digest),
CachedEdited.densityHash==EditedBase->densityHash?1:0,
CachedEdited.meshHash==EditedBase->meshHash?1:0,
CachedEdited.payloadHash==EditedBase->payloadHash?1:0,
CachedEdited.topologyHash==EditedBase->topologyHash?1:0,
bEditCacheSame?1:0,bEditCacheSame?"PASS":"FAIL",CacheError.c_str());
}
const bool bEditPass=Before&&Applied&&EditedBase&&bEditPageBuilt
&&bEditPageVerified&&bEditChanged&&bEditSame&&EditComparison.valid
&&EditComparison.exactQueryMismatches==0;
@@ -996,11 +1079,13 @@ namespace StandaloneStreamBenchmark
&&HistoryPairs==(bHasCameraStep&&bRegionCrossing?1:0)&&HistoryMismatches==0
&&PageRoutesVerified==static_cast<int32>(Rows.size())
&&NativeRoutesVerified==NativePairs&&PageProofViolations==0
&&PageExactMismatches==0&&NativeExactMismatches==0;
std::printf("[RouteParityAudit] seed=%d page_pairs=%zu page_verified=%d page_mismatches=%d native_only_pairs=%d native_verified=%d native_mismatches=%d higher_lod_exact_pairs=%d higher_lod_exact_mismatches=%d edit_pairs=%d edit_mismatches=%d history_pairs=%d history_mismatches=%d exact_query_samples=%llu exact_query_mismatches=%llu page_density_samples=%llu max_abs_density_delta=%.9g mean_abs_density_delta=%.9g sign_mismatches=%llu edge_crossing_mismatches=%llu shared_surface_edges=%llu max_surface_delta_vox=%.9g mean_surface_delta_vox=%.9g page_proof_blocks=%llu page_proof_samples=%llu page_proof_fallbacks=%llu page_proof_unknown=%llu page_proof_violations=%llu status=%s\n",
&&PageExactMismatches==0&&NativeExactMismatches==0
&&(!TileCache||(CachePairs>0&&CacheMismatches==0&&EditCachePairs==1&&EditCacheMismatches==0));
std::printf("[RouteParityAudit] seed=%d page_pairs=%zu page_verified=%d page_mismatches=%d native_only_pairs=%d native_verified=%d native_mismatches=%d higher_lod_exact_pairs=%d higher_lod_exact_mismatches=%d edit_pairs=%d edit_mismatches=%d history_pairs=%d history_mismatches=%d cache_pairs=%d cache_mismatches=%d edit_cache_pairs=%d edit_cache_mismatches=%d exact_query_samples=%llu exact_query_mismatches=%llu page_density_samples=%llu max_abs_density_delta=%.9g mean_abs_density_delta=%.9g sign_mismatches=%llu edge_crossing_mismatches=%llu shared_surface_edges=%llu max_surface_delta_vox=%.9g mean_surface_delta_vox=%.9g page_proof_blocks=%llu page_proof_samples=%llu page_proof_fallbacks=%llu page_proof_unknown=%llu page_proof_violations=%llu status=%s\n",
World.Seed,Rows.size(),PageRoutesVerified,BasePageMismatches,
NativePairs,NativeRoutesVerified,NativeMismatches,ExactQueryPairs,
ExactQueryMismatches,EditPairs,EditMismatches,HistoryPairs,HistoryMismatches,
CachePairs,CacheMismatches,EditCachePairs,EditCacheMismatches,
static_cast<unsigned long long>(PageExactSamples
+NativeExactSamples+ExactQuerySamples),
static_cast<unsigned long long>(PageExactMismatches
@@ -1640,10 +1725,12 @@ namespace StandaloneStreamBenchmark
const double WholeViewSeconds=std::chrono::duration<double>(WholeViewReady-ColdStart).count();
const double StreamSeconds=std::chrono::duration<double>(WholeViewReady-StreamStart).count();
const double FirstTileSeconds=std::chrono::duration<double>(FirstTileReady-ColdStart).count();
std::printf("[StreamPipeline] workers=%d shape=%.3fs first_tile_ready=%.3fs neighborhood_mesh=%.3fs neighborhood_ready=%.3fs full_detail_ready=%.3fs whole_view_ready=%.3fs neighborhood_tiles=%d/27 full_detail_tiles=%d/%d whole_tiles=%d/%zu stream_wall=%.3fs cache_hits=0 full_payload=1\n",
const std::uint64_t TileCacheHits=TileCache?TileCache->hitCount():Streamer.GetCacheHits();
std::printf("[StreamPipeline] workers=%d shape=%.3fs first_tile_ready=%.3fs neighborhood_mesh=%.3fs neighborhood_ready=%.3fs full_detail_ready=%.3fs whole_view_ready=%.3fs neighborhood_tiles=%d/27 full_detail_tiles=%d/%d whole_tiles=%d/%zu stream_wall=%.3fs cache_hits=%llu full_payload=1\n",
WorkerCount,ShapeSeconds,FirstTileSeconds,NeighborhoodMeshSeconds,NeighborhoodSeconds,
FullDetailSeconds,WholeViewSeconds,NeighborhoodFinished,
FullDetailFinished,FullDetailExpected,Finished,Desired.Tiles.size(),StreamSeconds);
FullDetailFinished,FullDetailExpected,Finished,Desired.Tiles.size(),StreamSeconds,
static_cast<unsigned long long>(TileCacheHits));
if(bReportMeshCoverage)
std::printf("[PipelineMeshCoverage] tiles=%d nonempty_mesh_tiles=%llu indexed_triangles=%llu\n",
Finished,static_cast<unsigned long long>(WholeNonEmptyMeshTiles),
@@ -1739,10 +1826,28 @@ namespace StandaloneStreamBenchmark
static_cast<unsigned long long>(ExactDetailHash),
static_cast<unsigned long long>(ExactCollisionHash));
}
bool bTileCacheSafetyPassing=true;
if(TileCache)
{
std::string CacheSafetyError;
bTileCacheSafetyPassing=TileCache->RunSafetyControls(CacheOptions,CacheSafetyError);
if(!bTileCacheSafetyPassing)
std::fprintf(stderr,"[TileCacheSafety] failed: %s\n",CacheSafetyError.c_str());
TileCache->PrintSummary();
TileCache->MeasureDiskReads();
const std::uint64_t CacheBytes=TileCache->bytesOnDisk();
std::printf("[TileCacheBudget] bytes=%llu maximum=%llu status=%s\n",
static_cast<unsigned long long>(CacheBytes),
static_cast<unsigned long long>(StandaloneTileCachePrototype::MaximumDiskBytes),
CacheBytes<=StandaloneTileCachePrototype::MaximumDiskBytes?"PASS":"FAIL");
bTileCacheSafetyPassing=bTileCacheSafetyPassing
&&CacheBytes<=StandaloneTileCachePrototype::MaximumDiskBytes;
}
return NeighborhoodFinished==static_cast<int32>(Neighborhood.size())
&&FullDetailFinished==FullDetailExpected&&Finished==Update.DesiredTiles&&bHashesPassing
&&(!bProfileColumnOverlap||bColumnOverlapCapturePassing)
&&(!bExportRequested||bSceneExportPassing)&&bScreenSpaceGatePassing
&&(!bAuditUniformBlockProofs||BlockProofViolations==0)?0:6;
&&(!bAuditUniformBlockProofs||BlockProofViolations==0)
&&bTileCacheSafetyPassing?0:6;
}
} // namespace StandaloneStreamBenchmark