wip(L16, side branch): disk terrain cache design + headless Linux prototype

Exact lossless zlib tile records (keyed on world/season/recipe/plan/evaluator/options/tile),
pristine cache plus ordered edits, server never trusts client caches, corruption/stale rejection,
atomic writes, 2 GB cap. Loaded tiles match regenerated hashes (L1 view, 24 m crossing).
NOT for assembler/world yet: the shared StreamBenchmarkPipeline.h includes Linux-only headers
(unistd/fcntl/zlib), which would break the Windows viewer build.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CSFYgsdW8ejZSyiMf4qRSX

Standalone/Source/StreamBenchmarkHeadless.cpp

@@ -35,6 +35,7 @@ namespace
         bool StreamBenchmark=false;
         bool StreamPipeline=false;
         bool StreamHash=false;
+        bool TileCachePrototype=false;
         bool ProfileColumnOverlap=false;
         bool UseSharedColumnCache=true;
         bool ForceGenerate=false;
@@ -119,6 +120,7 @@ namespace
                     "  --seed=N --recipe=surface-world|floating-islands\n"
                     "  --auto-surface-camera (place the 27-tile cube on this seed's surface)\n"
                     "  --mesh-coverage (report non-empty tiles and indexed triangles)\n"
+                    "  --tile-cache-prototype (opt-in exact disk-cache round trip; writes Saved/Linux/cache/l16-prototype)\n"
                     "  --screen-space-gate (measure display error and historical negative controls)\n"
                     "  --audit-uniform-block-proofs (fill and check every interval-certified block)\n"
                     "  --profile-lod-work (report per-LOD density sample counts)\n"
@@ -137,6 +139,15 @@ namespace
             else if(arg=="--stream-benchmark")options.StreamBenchmark=true;
             else if(arg=="--stream-pipeline")options.StreamPipeline=true;
             else if(arg=="--stream-hash")options.StreamHash=true;
+            else if(arg=="--tile-cache-prototype")
+            {
+                options.TileCachePrototype=true;
+                options.StreamPipeline=true;
+                options.StreamBenchmark=true;
+                options.WorldAssemblerPrototype=true;
+                options.ForceGenerate=true;
+                options.StreamHash=true;
+            }
             else if(arg=="--profile-column-overlap")options.ProfileColumnOverlap=true;
             else if(arg=="--force-generate")options.ForceGenerate=true;
             else if(arg=="--mesh-coverage")options.MeshCoverage=true;
@@ -377,5 +388,9 @@ int main(int argc,char** argv)
         cameraStepVoxels,options.AuditUniformBlockProofs,options.ProfileLodWork,
         options.RegionCrossing,options.AuditTileShapeReadiness,
         options.DiagnosticSlowShapeCompileMs,options.AuditBasePageEquivalence,
-        options.AuditRouteParity);
+        options.AuditRouteParity,
+        options.TileCachePrototype
+            ?std::filesystem::current_path()/"Saved/Linux/cache/l16-prototype"
+            :std::filesystem::path{},
+        options.DataPath);
 }

Standalone/Source/StreamBenchmarkPipeline.h

@@ -3,6 +3,7 @@
 #include "OwnerWorldLoader.h"
 #include "StreamingWorld.h"
 #include "LinuxScreenSpaceGate.h"
+#include "../Research/WorldAssemblerM3/Linux16/TileCachePrototype.h"
 
 #include <algorithm>
 #include <array>
@@ -253,7 +254,9 @@ namespace StandaloneStreamBenchmark
         bool bAuditUniformBlockProofs=false,bool bReportLodWorkProfile=false,
         bool bRegionCrossing=false,
         bool bAuditTileShapeReadiness=false,int DiagnosticSlowShapeCompileMs=0,
-        bool bAuditBasePageEquivalence=false,bool bAuditRouteParity=false)
+        bool bAuditBasePageEquivalence=false,bool bAuditRouteParity=false,
+        const std::filesystem::path& TileCachePrototypeRoot={},
+        const std::filesystem::path& OwnerSnapshotPath={})
     {
         using Clock=std::chrono::steady_clock;
         FStandaloneOwnerWorld& World=*OwnerWorld;
@@ -274,6 +277,42 @@ namespace StandaloneStreamBenchmark
         const bool bRoundTunnels=World.Generator->bStandaloneResearchRoundTunnels;
         const bool bEnhancedTunnelWalls=World.Generator->bStandaloneResearchEnhancedWalls;
         VoxelForgeCore::World Shape(CoreSnapshot);
+        std::unique_ptr<StandaloneTileCachePrototype::TileDiskCache> TileCache;
+        FStandaloneCoreTileRoute CoreTileRoute;
+        VoxelForgeCore::TileGenerationOptions CacheOptions;
+        if(!TileCachePrototypeRoot.empty())
+        {
+            const auto RepoRoot=std::filesystem::current_path().lexically_normal();
+            const auto AllowedRoot=(RepoRoot/"Saved/Linux/cache").lexically_normal();
+            const auto RequestedRoot=std::filesystem::absolute(TileCachePrototypeRoot).lexically_normal();
+            const std::string AllowedString=AllowedRoot.string()+std::string(1,std::filesystem::path::preferred_separator);
+            if(RequestedRoot.string().rfind(AllowedString,0)!=0||OwnerSnapshotPath.empty())
+            {
+                std::fprintf(stderr,"[TileCachePrototype] path rejected: cache writes must stay under Saved/Linux/cache and owner snapshot identity must be provided.\n");
+                return 2;
+            }
+            if(ClosestDisplayDistances[0]==0.0)
+                ClosestDisplayDistances=StandaloneScreenSpaceGate::ClosestDisplayedDistancesMeters(
+                    World,CameraPositionVoxels,World.CoreVoxelSizeMicrons);
+            const auto Identity=StandaloneTileCachePrototype::MakeIdentity(
+                World,CoreSnapshot,OwnerSnapshotPath);
+            if(Identity.ownerSnapshotHash==0||!Assembly)
+            {
+                std::fprintf(stderr,"[TileCachePrototype] missing owner snapshot or world assembly identity.\n");
+                return 2;
+            }
+            TileCache=std::make_unique<StandaloneTileCachePrototype::TileDiskCache>(
+                RequestedRoot,Identity,ClosestDisplayDistances);
+            CacheOptions.hashDensity=true;
+            StandaloneTileCachePrototype::TileDiskCache* Cache=TileCache.get();
+            CoreTileRoute=[Cache](const VoxelForgeCore::World& TileWorld,
+                const VoxelForgeCore::TileKey& Key,
+                const VoxelForgeCore::TileGenerationOptions& Options,bool& bHit,
+                std::string& Error)
+            {return Cache->GenerateOrLoad(TileWorld,Key,Options,bHit,Error);};
+            std::printf("[TileCachePrototype] enabled root=%s owner_snapshot=%s mode=worker_generate_store_read_or_warm_hit codec=zlib-6\n",
+                RequestedRoot.string().c_str(),OwnerSnapshotPath.string().c_str());
+        }
         FStandaloneDesiredTileSet Desired=StandaloneWorldMesher::BuildDesiredSet(World,CameraPositionVoxels);
         const auto CoreTiles=Shape.desiredTiles(
             {CameraPositionVoxels.X,CameraPositionVoxels.Y,CameraPositionVoxels.Z});
@@ -411,7 +450,7 @@ namespace StandaloneStreamBenchmark
         ShapeRequest.enhancedTunnelWalls=bEnhancedTunnelWalls;
         FStandaloneStreamingGenerator Streamer(World,WorkerCount,bHashDensity,nullptr,MaxLevel,
             bSkipUvAndColors,bSkipMaterialGroups,bSkipUV,bSkipColors,&Shape,bProfileColumnOverlap,
-            FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache);
+            FarHorizonLOD8InteriorStepVoxels,bUseSharedColumnCache,std::move(CoreTileRoute));
         Streamer.SetUniformBlockProofAuditEnabled(bAuditUniformBlockProofs);
         if(bScreenSpaceGate||bReportLodWorkProfile)Streamer.SetMesherStageProfilingEnabled(true);
         const auto StreamStart=Clock::now();
@@ -657,6 +696,7 @@ namespace StandaloneStreamBenchmark
 
             VoxelForgeCore::TileGenerationOptions PageOptions=CaptureOptions;
             PageOptions.auditUniformBlockProofs=true;
+            int32 CachePairs=0,CacheMismatches=0,EditCachePairs=0,EditCacheMismatches=0;
             int32 BasePageMismatches=0,PageRoutesVerified=0;
             std::uint64_t PageProofBlocks=0,PageProofSamples=0,PageProofFallbacks=0,
                 PageProofUnknown=0,PageProofViolations=0;
@@ -716,6 +756,28 @@ namespace StandaloneStreamBenchmark
                     PageProofUnknown+=Page->diagnostics.uniformBlockProofUnknownBlocks;
                     PageProofViolations+=Page->diagnostics.uniformBlockProofViolations;
                 }
+                if(TileCache)
+                {
+                    VoxelForgeCore::TileMesh Cached;
+                    std::string CacheError;
+                    bool bCacheReady=TileCache->LoadExpected(Row.Candidate.Key,CacheOptions,Cached,CacheError);
+                    if(!bCacheReady&&Row.Base)
+                        bCacheReady=TileCache->StoreAndLoadExplicit(*Row.Base,CacheOptions,
+                            TileCache->identity(),Cached,CacheError);
+                    const bool bCacheSame=bCacheReady&&Row.Base
+                        &&SameTileOutput(*Row.Base,Cached)
+                        &&StandaloneTileCachePrototype::SameExactMesh(*Row.Base,Cached);
+                    ++CachePairs;
+                    if(!bCacheSame)++CacheMismatches;
+                    std::printf("[RouteParityPair] route=base/disk-cache key=(%d,%d,%d,lod%d) same_density=%d same_mesh=%d same_payload=%d same_topology=%d exact_arrays=%d status=%s error=%s\n",
+                        Row.Candidate.Key.coordinate.x,Row.Candidate.Key.coordinate.y,
+                        Row.Candidate.Key.coordinate.z,Row.Candidate.Key.lod,
+                        Row.Base&&Cached.densityHash==Row.Base->densityHash?1:0,
+                        Row.Base&&Cached.meshHash==Row.Base->meshHash?1:0,
+                        Row.Base&&Cached.payloadHash==Row.Base->payloadHash?1:0,
+                        Row.Base&&Cached.topologyHash==Row.Base->topologyHash?1:0,
+                        bCacheSame?1:0,bCacheSame?"PASS":"FAIL",CacheError.c_str());
+                }
                 std::printf("[RouteParityPair] route=base/page key=(%d,%d,%d,lod%d) page_requested=%d page_verified=%d same_density=%d same_mesh=%d same_payload=%d same_topology=%d samples=%llu max_abs_density_delta=%.9g sign_mismatches=%llu edge_crossing_mismatches=%llu shared_surface_edges=%llu max_surface_delta_vox=%.9g exact_query_mismatches=%llu base_native_float=%llu page_native_float=%llu first_density=(%.9g,%.9g) status=%s error=%s\n",
                     Row.Candidate.Key.coordinate.x,Row.Candidate.Key.coordinate.y,
                     Row.Candidate.Key.coordinate.z,Row.Candidate.Key.lod,
@@ -899,6 +961,27 @@ namespace StandaloneStreamBenchmark
                         &&EditedPage->diagnostics.analyticRegionPageRoute;
                     const bool bEditSame=EditedBase&&EditedPage&&bEditPageVerified
                         &&SameTileOutput(*EditedBase,*EditedPage);
+                    bool bEditCacheSame=false;
+                    if(TileCache&&EditedBase&&Applied)
+                    {
+                        VoxelForgeCore::TileMesh CachedEdited;
+                        std::string CacheError;
+                        const auto EditedIdentity=TileCache->WithEdit(Transaction.sequence,Applied->digest);
+                        bEditCacheSame=TileCache->StoreAndLoadExplicit(*EditedBase,CacheOptions,
+                            EditedIdentity,CachedEdited,CacheError)
+                            &&SameTileOutput(*EditedBase,CachedEdited);
+                        ++EditCachePairs;
+                        if(!bEditCacheSame)++EditCacheMismatches;
+                        std::printf("[RouteParityPair] route=edited-base/disk-cache key=(%d,%d,%d,lod%d) edit_sequence=%llu edit_digest=0x%016llx same_density=%d same_mesh=%d same_payload=%d same_topology=%d exact_arrays=%d status=%s error=%s\n",
+                            Key.coordinate.x,Key.coordinate.y,Key.coordinate.z,Key.lod,
+                            static_cast<unsigned long long>(Transaction.sequence),
+                            static_cast<unsigned long long>(Applied->digest),
+                            CachedEdited.densityHash==EditedBase->densityHash?1:0,
+                            CachedEdited.meshHash==EditedBase->meshHash?1:0,
+                            CachedEdited.payloadHash==EditedBase->payloadHash?1:0,
+                            CachedEdited.topologyHash==EditedBase->topologyHash?1:0,
+                            bEditCacheSame?1:0,bEditCacheSame?"PASS":"FAIL",CacheError.c_str());
+                    }
                     const bool bEditPass=Before&&Applied&&EditedBase&&bEditPageBuilt
                         &&bEditPageVerified&&bEditChanged&&bEditSame&&EditComparison.valid
                         &&EditComparison.exactQueryMismatches==0;
@@ -996,11 +1079,13 @@ namespace StandaloneStreamBenchmark
                 &&HistoryPairs==(bHasCameraStep&&bRegionCrossing?1:0)&&HistoryMismatches==0
                 &&PageRoutesVerified==static_cast<int32>(Rows.size())
                 &&NativeRoutesVerified==NativePairs&&PageProofViolations==0
-                &&PageExactMismatches==0&&NativeExactMismatches==0;
-            std::printf("[RouteParityAudit] seed=%d page_pairs=%zu page_verified=%d page_mismatches=%d native_only_pairs=%d native_verified=%d native_mismatches=%d higher_lod_exact_pairs=%d higher_lod_exact_mismatches=%d edit_pairs=%d edit_mismatches=%d history_pairs=%d history_mismatches=%d exact_query_samples=%llu exact_query_mismatches=%llu page_density_samples=%llu max_abs_density_delta=%.9g mean_abs_density_delta=%.9g sign_mismatches=%llu edge_crossing_mismatches=%llu shared_surface_edges=%llu max_surface_delta_vox=%.9g mean_surface_delta_vox=%.9g page_proof_blocks=%llu page_proof_samples=%llu page_proof_fallbacks=%llu page_proof_unknown=%llu page_proof_violations=%llu status=%s\n",
+                &&PageExactMismatches==0&&NativeExactMismatches==0
+                &&(!TileCache||(CachePairs>0&&CacheMismatches==0&&EditCachePairs==1&&EditCacheMismatches==0));
+            std::printf("[RouteParityAudit] seed=%d page_pairs=%zu page_verified=%d page_mismatches=%d native_only_pairs=%d native_verified=%d native_mismatches=%d higher_lod_exact_pairs=%d higher_lod_exact_mismatches=%d edit_pairs=%d edit_mismatches=%d history_pairs=%d history_mismatches=%d cache_pairs=%d cache_mismatches=%d edit_cache_pairs=%d edit_cache_mismatches=%d exact_query_samples=%llu exact_query_mismatches=%llu page_density_samples=%llu max_abs_density_delta=%.9g mean_abs_density_delta=%.9g sign_mismatches=%llu edge_crossing_mismatches=%llu shared_surface_edges=%llu max_surface_delta_vox=%.9g mean_surface_delta_vox=%.9g page_proof_blocks=%llu page_proof_samples=%llu page_proof_fallbacks=%llu page_proof_unknown=%llu page_proof_violations=%llu status=%s\n",
                 World.Seed,Rows.size(),PageRoutesVerified,BasePageMismatches,
                 NativePairs,NativeRoutesVerified,NativeMismatches,ExactQueryPairs,
                 ExactQueryMismatches,EditPairs,EditMismatches,HistoryPairs,HistoryMismatches,
+                CachePairs,CacheMismatches,EditCachePairs,EditCacheMismatches,
                 static_cast<unsigned long long>(PageExactSamples
                     +NativeExactSamples+ExactQuerySamples),
                 static_cast<unsigned long long>(PageExactMismatches
@@ -1640,10 +1725,12 @@ namespace StandaloneStreamBenchmark
         const double WholeViewSeconds=std::chrono::duration<double>(WholeViewReady-ColdStart).count();
         const double StreamSeconds=std::chrono::duration<double>(WholeViewReady-StreamStart).count();
         const double FirstTileSeconds=std::chrono::duration<double>(FirstTileReady-ColdStart).count();
-        std::printf("[StreamPipeline] workers=%d shape=%.3fs first_tile_ready=%.3fs neighborhood_mesh=%.3fs neighborhood_ready=%.3fs full_detail_ready=%.3fs whole_view_ready=%.3fs neighborhood_tiles=%d/27 full_detail_tiles=%d/%d whole_tiles=%d/%zu stream_wall=%.3fs cache_hits=0 full_payload=1\n",
+        const std::uint64_t TileCacheHits=TileCache?TileCache->hitCount():Streamer.GetCacheHits();
+        std::printf("[StreamPipeline] workers=%d shape=%.3fs first_tile_ready=%.3fs neighborhood_mesh=%.3fs neighborhood_ready=%.3fs full_detail_ready=%.3fs whole_view_ready=%.3fs neighborhood_tiles=%d/27 full_detail_tiles=%d/%d whole_tiles=%d/%zu stream_wall=%.3fs cache_hits=%llu full_payload=1\n",
             WorkerCount,ShapeSeconds,FirstTileSeconds,NeighborhoodMeshSeconds,NeighborhoodSeconds,
             FullDetailSeconds,WholeViewSeconds,NeighborhoodFinished,
-            FullDetailFinished,FullDetailExpected,Finished,Desired.Tiles.size(),StreamSeconds);
+            FullDetailFinished,FullDetailExpected,Finished,Desired.Tiles.size(),StreamSeconds,
+            static_cast<unsigned long long>(TileCacheHits));
         if(bReportMeshCoverage)
             std::printf("[PipelineMeshCoverage] tiles=%d nonempty_mesh_tiles=%llu indexed_triangles=%llu\n",
                 Finished,static_cast<unsigned long long>(WholeNonEmptyMeshTiles),
@@ -1739,10 +1826,28 @@ namespace StandaloneStreamBenchmark
                 static_cast<unsigned long long>(ExactDetailHash),
                 static_cast<unsigned long long>(ExactCollisionHash));
         }
+        bool bTileCacheSafetyPassing=true;
+        if(TileCache)
+        {
+            std::string CacheSafetyError;
+            bTileCacheSafetyPassing=TileCache->RunSafetyControls(CacheOptions,CacheSafetyError);
+            if(!bTileCacheSafetyPassing)
+                std::fprintf(stderr,"[TileCacheSafety] failed: %s\n",CacheSafetyError.c_str());
+            TileCache->PrintSummary();
+            TileCache->MeasureDiskReads();
+            const std::uint64_t CacheBytes=TileCache->bytesOnDisk();
+            std::printf("[TileCacheBudget] bytes=%llu maximum=%llu status=%s\n",
+                static_cast<unsigned long long>(CacheBytes),
+                static_cast<unsigned long long>(StandaloneTileCachePrototype::MaximumDiskBytes),
+                CacheBytes<=StandaloneTileCachePrototype::MaximumDiskBytes?"PASS":"FAIL");
+            bTileCacheSafetyPassing=bTileCacheSafetyPassing
+                &&CacheBytes<=StandaloneTileCachePrototype::MaximumDiskBytes;
+        }
         return NeighborhoodFinished==static_cast<int32>(Neighborhood.size())
             &&FullDetailFinished==FullDetailExpected&&Finished==Update.DesiredTiles&&bHashesPassing
             &&(!bProfileColumnOverlap||bColumnOverlapCapturePassing)
             &&(!bExportRequested||bSceneExportPassing)&&bScreenSpaceGatePassing
-            &&(!bAuditUniformBlockProofs||BlockProofViolations==0)?0:6;
+            &&(!bAuditUniformBlockProofs||BlockProofViolations==0)
+            &&bTileCacheSafetyPassing?0:6;
     }
 } // namespace StandaloneStreamBenchmark

Standalone/Source/StreamingWorld.h

@@ -67,6 +67,10 @@ struct FStandaloneCpuTile
 
 };
 
+using FStandaloneCoreTileRoute = std::function<std::optional<VoxelForgeCore::TileMesh>(
+    const VoxelForgeCore::World&,const VoxelForgeCore::TileKey&,
+    const VoxelForgeCore::TileGenerationOptions&,bool&,std::string&)>;
+
 inline uint64 HashStandaloneMeshPayload(const FVoxelMeshData& Mesh)
 {
     uint64 Hash=14695981039346656037ull;
@@ -430,14 +434,16 @@ public:
                                            VoxelForgeCore::World* InCoreWorld=nullptr,
                                            bool bInCaptureColumnCoordinates=false,
                                            int32 InLod8HorizonInteriorStepVoxels=0,
-                                           bool bInSharedColumnCache=true)
+                                           bool bInSharedColumnCache=true,
+                                           FStandaloneCoreTileRoute InCoreTileRoute={})
         :World(&InWorld),WorkerCount(FMath::Clamp(InWorkerCount,1,16)),bHashDensity(bInHashDensity),
          MeshCache(InMeshCache),MaxLevelFilter(InMaxLevelFilter),
          CoreWorld(InCoreWorld),bSkipUvAndColors(bInSkipUvAndColors),bSkipMaterialGroups(bInSkipMaterialGroups),
          bSkipUV(bInSkipUV),bSkipColors(bInSkipColors),
          bCaptureColumnCoordinates(bInCaptureColumnCoordinates),
          bSharedColumnCache(bInSharedColumnCache),
-         Lod8HorizonInteriorStepVoxels(std::max(0,InLod8HorizonInteriorStepVoxels))
+         Lod8HorizonInteriorStepVoxels(std::max(0,InLod8HorizonInteriorStepVoxels)),
+         CoreTileRoute(std::move(InCoreTileRoute))
     {
         VoxelGenLOD::InitializeConsoleSwitches();
         UVoxelMarchingCubesMesher::InitializeConsoleSwitches();
@@ -830,8 +836,11 @@ private:
             Options.diagnosticColumnInterpolationStride=
                 std::max<std::uint32_t>(1,DiagnosticCellStride);
             std::string Error;
-            auto Result=CoreWorld->generateTile(
-                {{Job.Key.Coord.X,Job.Key.Coord.Y,Job.Key.Coord.Z},Job.Key.Level},Options,Error);
+            const VoxelForgeCore::TileKey CoreKey{{Job.Key.Coord.X,Job.Key.Coord.Y,Job.Key.Coord.Z},Job.Key.Level};
+            bool bDiskCacheHit=false;
+            auto Result=CoreTileRoute
+                ?CoreTileRoute(*CoreWorld,CoreKey,Options,bDiskCacheHit,Error)
+                :CoreWorld->generateTile(CoreKey,Options,Error);
             if(!Result)
             {
                 std::fprintf(stderr,"[VoxelForgeCore] tile generation failed: %s\n",Error.c_str());
@@ -1003,6 +1012,7 @@ private:
     bool bCaptureColumnCoordinates=false;
     bool bSharedColumnCache=true;
     int32 Lod8HorizonInteriorStepVoxels=0;
+    FStandaloneCoreTileRoute CoreTileRoute;
     bool bProfileMesherStages=false;
     bool bAuditUniformBlockProofs=false;
     std::atomic<uint64> CacheHitCount{0};