diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1b7d232..9bc4a22 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2162,6 +2162,28 @@ recipe/style audits, route support, open-air, strict layout, legacy validators, paired-cost results are in [`Standalone/Research/WorldAssemblerM3/Linux36/REPORT.md`](Standalone/Research/WorldAssemblerM3/Linux36/REPORT.md). +### 8.50 Immutable field binding — World Assembler M3 L40 + +`FStandaloneShapeWorld` now keeps a sorted `FBoundStratumField` table built from the resolved layout +at world construction. Each entry owns a concrete SurfaceWorld, FloatingIslands, or legacy adapter +in `FGeneratorFieldAdapterV1`, plus the resolved bounds and native plan pointer. The adapters share +the immutable `densityAt(x,y,z,out)` shape, while `std::variant` keeps dispatch concrete instead of +adding a virtual call to every density sample. FloatingIslands keeps its original fixed-point entry +point for sample evaluation and its boundary parameter table is captured when the adapter is bound. + +`DensityAt`, native SurfaceWorld/FloatingIslands evaluation, legacy SurfaceWorld bridging and +`ClassifyBoxDetailed` select from the bound table. The native adapters no longer ask the manager +which generator or plan to use per sample; box proof reuses the bound field objects. The existing +float-to-fixed conversion order, density arithmetic and shared boundary composition remain intact. +The manager's layout-width value is captured once for the legacy-compatible chunk-index projection. + +The legacy adapter is still a compatibility bridge: `UVoxelGenerator::GetDensityAt` retains its +own manager routing when invoked for legacy SurfaceWorld fields or out-of-plan fallback queries. +Other shared-composition policy branches and passage/boundary planning remain outside this slice. +No Unreal adapter or `VoxelWorld.cpp` change was needed, and the evaluator version is unchanged. +L37/L39 owner hashes are the parity baseline; the L40 report records fast, medium, full and paired +cost evidence. + ## 9. Multiplayer model (listen-server first, dedicated-friendly) > **Status: transport is DESIGN ONLY — nothing is networked in-tree yet** (no `Replicated`/`HasAuthority`/RPCs; a diff --git a/CODEMAP.md b/CODEMAP.md index 4a8241f..23a8cea 100644 --- a/CODEMAP.md +++ b/CODEMAP.md @@ -1268,6 +1268,17 @@ ARCHITECTURE §8.47 and `WorldAssemblyEvaluatorVersion=23`. See ARCHITECTURE §8.49 and [`Standalone/Research/WorldAssemblerM3/Linux36/REPORT.md`](Standalone/Research/WorldAssemblerM3/Linux36/REPORT.md). +### 3.33 Immutable field binding — World Assembler M3 L40 + +| Symbol | Canonical file | Role | +|---|---|---| +| `FSurfaceWorldFieldAdapter`, `FFloatingIslandFieldAdapter`, `FLegacyGeneratorFieldAdapter`, `FGeneratorFieldAdapterV1` | `Standalone/Core/src/ShapeWorldInternal.h` | Concrete, immutable density adapters with one shared `densityAt(x,y,z,out)` call shape; the variant avoids per-sample virtual dispatch. FI retains its fixed-point entry point and prebound legacy boundary parameters. | +| `FBoundStratumField`, `BindResolvedStratumFields`, `FindBoundStratumForChunk` | `Standalone/Core/src/ShapeWorldInternal.h` | Bind a resolved strate's plan, evaluator and chunk interval once during world initialization; native field and classification queries select this table. | +| `NativeSurfaceDensityAt`, `NativeFloatingIslandDensityAt`, `ClassifyBoxDetailed` | `Standalone/Core/src/ShapeWorldInternal.h` | Consume the prebound adapters and plan pointers instead of looking up strate type/parameters through the manager per query. | + +Hash parity is unchanged from L37/L39; see ARCHITECTURE §8.50 and +[`Standalone/Research/WorldAssemblerM3/Linux40/REPORT.md`](Standalone/Research/WorldAssemblerM3/Linux40/REPORT.md). + ## 4. The density pipeline (most-edited hot path) ### 4.1 `GetDensityWithParams` (TunnelNetwork) — VoxelGenerator.cpp:3749 diff --git a/Standalone/Core/src/ShapeWorldInternal.h b/Standalone/Core/src/ShapeWorldInternal.h index dc1b263..c19988f 100644 --- a/Standalone/Core/src/ShapeWorldInternal.h +++ b/Standalone/Core/src/ShapeWorldInternal.h @@ -25,6 +25,7 @@ #include #include #include +#include #include // Standalone-only world compiler. It compiles the runtime room/tunnel/passages into @@ -41,6 +42,114 @@ public: double SealThickness=0.0; }; + // Immutable adapters share densityAt(x,y,z,out) without a virtual call in the sample + // loop. The variant is selected once per resolved strate; callers retain concrete access + // to fixed-point and interval operations where the legacy arithmetic requires them. + struct FSurfaceWorldFieldAdapter + { + VoxelForgeCore::Heightfield2D Evaluator; + FSurfaceWorldFieldAdapter(const VoxelForgeCore::WorldAssemblyRecipeV1& Recipe, + const VoxelForgeCore::WorldAssemblyStratumPlanV1& Plan) noexcept + :Evaluator(Recipe,Plan){} + + bool densityAt(float X,float Y,float Z,float& OutDensity) const noexcept + { + if(!std::isfinite(X)||!std::isfinite(Y)||!std::isfinite(Z))return false; + constexpr double Scale=static_cast( + VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel); + const double MaxVoxel=static_cast(std::numeric_limits::max())/Scale; + if(std::abs(static_cast(X))>MaxVoxel + ||std::abs(static_cast(Y))>MaxVoxel + ||std::abs(static_cast(Z))>MaxVoxel)return false; + const auto ToFixed=[](float Value) + {return static_cast(std::llround(static_cast(Value) + *VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel));}; + if(!Evaluator.isValid())return false; + OutDensity=static_cast(Evaluator.point(ToFixed(X),ToFixed(Y),ToFixed(Z)) + .densitySubvoxels/Scale); + return true; + } + }; + + struct FFloatingIslandFieldAdapter + { + VoxelForgeCore::FloatingIslandFieldV1 Evaluator; + int32 BottomChunkZ=0; + std::vector LegacyBoundaryParams; + + FFloatingIslandFieldAdapter(const VoxelForgeCore::FloatingIslandRecipeV1& Recipe, + const VoxelForgeCore::FloatingIslandPlanV1& Plan,int32 InBottomChunkZ, + int32 InTopChunkZ,const UVoxelStrateManager* Manager) + :Evaluator(Recipe,Plan),BottomChunkZ(InBottomChunkZ) + { + if(Manager&&Recipe.geometryVersion<4&&InTopChunkZ>=InBottomChunkZ) + { + LegacyBoundaryParams.reserve(static_cast( + static_cast(InTopChunkZ)-InBottomChunkZ+1)); + for(int32 ChunkZ=InBottomChunkZ;ChunkZ<=InTopChunkZ;++ChunkZ) + LegacyBoundaryParams.push_back(Manager->GetFloatingIslandParamsForChunk( + FIntVector(0,0,ChunkZ))); + } + } + + std::int64_t densitySubvoxelsAt(std::int64_t X,std::int64_t Y, + std::int64_t Z) const noexcept + {return Evaluator.densitySubvoxels(X,Y,Z);} + + const FFloatingIslandParams* legacyBoundaryParamsAt(int32 ChunkZ) const noexcept + { + const int64 Index=static_cast(ChunkZ)-BottomChunkZ; + return Index>=0&&static_cast(Index)(Index)]:nullptr; + } + + bool densityAt(float X,float Y,float Z,float& OutDensity) const noexcept + { + if(!Evaluator.isValid()||!std::isfinite(X)||!std::isfinite(Y) + ||!std::isfinite(Z))return false; + constexpr double Scale=static_cast( + VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel); + constexpr double Maximum=1000000000000.0/Scale; + if(std::abs(static_cast(X))>Maximum + ||std::abs(static_cast(Y))>Maximum + ||std::abs(static_cast(Z))>Maximum)return false; + const auto ToFixed=[](float Value) + {return static_cast(std::llround(static_cast(Value) + *VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel));}; + OutDensity=static_cast(static_cast(densitySubvoxelsAt( + ToFixed(X),ToFixed(Y),ToFixed(Z)))/Scale); + return std::isfinite(OutDensity); + } + }; + + struct FLegacyGeneratorFieldAdapter + { + const UVoxelGenerator* Generator=nullptr; + bool densityAt(float X,float Y,float Z,float& OutDensity) const + { + if(!Generator)return false; + OutDensity=Generator->GetDensityAt(X,Y,Z); + return std::isfinite(OutDensity); + } + }; + + using FGeneratorFieldAdapterV1=std::variant; + + struct FBoundStratumField + { + enum class EKind : uint8 { Legacy, SurfaceWorld, FloatingIslands }; + const VoxelForgeCore::ResolvedLayoutStratumV2* Resolved=nullptr; + const VoxelForgeCore::WorldAssemblyStratumPlanV1* SurfacePlan=nullptr; + const VoxelForgeCore::WorldAssemblyFloatingIslandPlanV1* IslandPlan=nullptr; + FGeneratorFieldAdapterV1 Evaluator; + int32 StrateIndex=-1; + int32 BottomChunkZ=0; + int32 TopChunkZ=-1; + EKind Kind=EKind::Legacy; + }; + const std::vector& GetPassageVoidCrossings() const {return PassageVoidCrossings;} @@ -170,6 +279,117 @@ public: return it==ResolvedLayout->strates.end()?nullptr:&*it; } + void BindResolvedStratumFields() + { + BoundStratumFields.clear(); + BoundManagerLayoutChunkWidthVoxels=Owner&&Owner->Manager + ?Owner->Manager->GetStrateLayoutChunkWidthVoxels():0.0f; + if(!ResolvedLayout||ResolvedLayout->strateLayoutChunkSizeMicrometres<=0)return; + const int64 ChunkWidthMicrometres=ResolvedLayout->strateLayoutChunkSizeMicrometres; + const auto FloorDiv64=[](int64 Value,int64 Divisor) + { + const int64 Quotient=Value/Divisor; + const int64 Remainder=Value%Divisor; + return Remainder<0?Quotient-1:Quotient; + }; + BoundStratumFields.reserve(ResolvedLayout->strates.size()); + for(const auto& Stratum:ResolvedLayout->strates) + { + if(Stratum.strateIndex<0 + ||Stratum.interiorTopMicrometres<=Stratum.interiorBottomMicrometres) + continue; + const int64 Bottom=FloorDiv64(Stratum.interiorBottomMicrometres, + ChunkWidthMicrometres); + const int64 Top=FloorDiv64(Stratum.interiorTopMicrometres-1, + ChunkWidthMicrometres); + if(Bottom::min() + ||Bottom>std::numeric_limits::max() + ||Top::min() + ||Top>std::numeric_limits::max()||Top(Bottom); + Bound.TopChunkZ=static_cast(Top); + if(Stratum.generatorKind=="SurfaceWorld") + { + Bound.Kind=FBoundStratumField::EKind::SurfaceWorld; + if(WorldAssembly) + { + const auto Plan=std::find_if(WorldAssembly->plan.strates.begin(), + WorldAssembly->plan.strates.end(),[&](const auto& Candidate) + {return Candidate.strateIndex==Bound.StrateIndex + &&Candidate.bottomChunkZ==Bound.BottomChunkZ + &&Candidate.topChunkZ==Bound.TopChunkZ;}); + if(Plan!=WorldAssembly->plan.strates.end()) + { + Bound.SurfacePlan=&*Plan; + FSurfaceWorldFieldAdapter Adapter(WorldAssembly->recipe,*Plan); + if(Adapter.Evaluator.isValid()) + Bound.Evaluator.emplace( + std::move(Adapter)); + } + } + } + else if(Stratum.generatorKind=="FloatingIslands") + { + Bound.Kind=FBoundStratumField::EKind::FloatingIslands; + if(WorldAssembly) + { + const auto Plan=std::find_if(WorldAssembly->floatingIslands.begin(), + WorldAssembly->floatingIslands.end(),[&](const auto& Candidate) + {return Candidate.strateIndex==Bound.StrateIndex + &&Candidate.bottomChunkZ==Bound.BottomChunkZ + &&Candidate.topChunkZ==Bound.TopChunkZ;}); + if(Plan!=WorldAssembly->floatingIslands.end()) + { + Bound.IslandPlan=&*Plan; + Bound.Evaluator.emplace( + Plan->manifest.recipe,Plan->manifest.plan, + Bound.BottomChunkZ,Bound.TopChunkZ, + Owner?Owner->Manager.get():nullptr); + } + } + } + if(std::holds_alternative(Bound.Evaluator)) + Bound.Evaluator.emplace( + FLegacyGeneratorFieldAdapter{Owner?Owner->Generator.get():nullptr}); + BoundStratumFields.push_back(std::move(Bound)); + } + std::sort(BoundStratumFields.begin(),BoundStratumFields.end(), + [](const auto& A,const auto& B){return A.BottomChunkZ(GetResolvedLayoutChunkWidthVoxels()); + if(!(LayoutWidth>0.0f))return nullptr; + return FindBoundStratumForChunk(FMath::FloorToInt(Z/LayoutWidth)); + } + + const FBoundStratumField* FindBoundStratumForManagerZ(float Z) const + { + if(!std::isfinite(Z)||!Owner||!(BoundManagerLayoutChunkWidthVoxels>0.0f)) + return nullptr; + const float WorldZCentimetres=static_cast( + static_cast(Z)*GetCoreVoxelSizeCm()); + const int32 ChunkZ=FMath::FloorToInt((WorldZCentimetres/VOXEL_SIZE) + /BoundManagerLayoutChunkWidthVoxels); + return FindBoundStratumForChunk(ChunkZ); + } + const VoxelForgeCore::ResolvedLayoutStratumV2* FindResolvedLayoutStratumByOrder(int32 Order) const { if(!ResolvedLayout||Order<0)return nullptr; @@ -479,10 +699,9 @@ public: // samples. The merged intervals are conservative, so a positive is safe. return true; } - if(!ResolvedLayout||!Owner||!Owner->Manager||!std::isfinite(Z))return false; - const double worldZCentimetres=static_cast(Z)*GetCoreVoxelSizeCm(); - const int32 strateIndex=Owner->Manager->GetStrateIndex(static_cast(worldZCentimetres)); - const auto* stratum=FindResolvedLayoutStratum(strateIndex); + if(!ResolvedLayout||!Owner||!std::isfinite(Z))return false; + const FBoundStratumField* bound=FindBoundStratumForManagerZ(Z); + const auto* stratum=bound?FindResolvedLayoutStratum(bound->StrateIndex):nullptr; if(!stratum)return false; const double zMicrometres=static_cast(Z)*Owner->CoreVoxelSizeMicrons; const auto solidAt=[&](const VoxelForgeCore::ResolvedLayoutBandV2& band) @@ -538,7 +757,7 @@ public: return density; } } - if(!ResolvedLayout||!Owner||!Owner->Manager||Owner->CoreVoxelSizeMicrons==0 + if(!ResolvedLayout||!Owner||Owner->CoreVoxelSizeMicrons==0 ||!std::isfinite(Z))return density; const double zMicrometres=static_cast(Z)*Owner->CoreVoxelSizeMicrons; if(ResolvedLayout->strates.size()==2) @@ -600,9 +819,8 @@ public: } } } - const double worldZCentimetres=static_cast(Z)*GetCoreVoxelSizeCm(); - const int32 strateIndex=Owner->Manager->GetStrateIndex(static_cast(worldZCentimetres)); - const auto* stratum=FindResolvedLayoutStratum(strateIndex); + const FBoundStratumField* bound=FindBoundStratumForManagerZ(Z); + const auto* stratum=bound?FindResolvedLayoutStratum(bound->StrateIndex):nullptr; if(!stratum)return density; const auto solidAt=[&](const VoxelForgeCore::ResolvedLayoutBandV2& band) {return band.presentInCurrentField&&zMicrometres>=band.bottomMicrometres @@ -1665,7 +1883,7 @@ public: bool GetResolvedStratumBounds(const FStrateSlot& Slot,FResolvedStratumBounds& Out) const { - if(!Owner||!Owner->Manager)return false; + if(!Owner)return false; if(const auto* resolved=FindResolvedLayoutStratum(Slot.StrateIndex)) { const double coreMicrons=std::max(1,Owner->CoreVoxelSizeMicrons); @@ -1678,11 +1896,11 @@ public: return MakeResolvedStratumBounds(bottom,top,std::max(ceiling,bedrock),Out); } const int32 MidChunkZ=(Slot.TopChunkZ+Slot.BottomChunkZ)/2; - if(bUseNativeManifestFields&&WorldAssembly - &&Owner->Manager->GetGeneratorTypeForChunk(FIntVector(0,0,MidChunkZ)) - ==ECaveGeneratorType::FloatingIslands) + const FBoundStratumField* Bound=FindBoundStratumForChunk(MidChunkZ); + if(bUseNativeManifestFields&&WorldAssembly&&Bound + &&Bound->Kind==FBoundStratumField::EKind::FloatingIslands) { - const auto* Plan=FindWorldAssemblyFloatingIslandStratum(Slot.StrateIndex,MidChunkZ); + const auto* Plan=Bound->IslandPlan; if(Plan&&Plan->manifest.recipe.geometryVersion>=3) { constexpr double Scale=static_cast( @@ -1989,6 +2207,7 @@ public: WorkerCount=std::clamp(Workers,1,16);DetailVariant=Variant; Scheduler=InScheduler;RoundTunnelsEnabled=bInRoundTunnels; bUseNativeManifestFields=true; + BindResolvedStratumFields(); bAnalyticOnly=false; bProfileDetailBand=bInProfileDetailBand; for(int32 A=0;A<3;++A) @@ -2162,6 +2381,7 @@ public: // Analytic pages retain their bounded owner bins for route bookkeeping. Once published, // the Core world binds each page to its base shape as the single density evaluator. bUseNativeManifestFields=true; + BindResolvedStratumFields(); RoundTunnelsEnabled=bInRoundTunnels;bAnalyticOnly=true;bProfileDetailBand=false; for(int32 A=0;A<3;++A) { @@ -2199,6 +2419,7 @@ public: Owner=&World;WorldAssembly=InWorldAssembly;ResolvedLayout=InResolvedLayout; NativeOnlyMinimum=InMinimum;NativeOnlyMaximum=InMaximum; bUseNativeManifestFields=true;bNativeOnly=true;bAnalyticOnly=true;Built=true; + BindResolvedStratumFields(); NativeOnlyViolations.store(0,std::memory_order_relaxed); } @@ -2742,23 +2963,13 @@ public: if(NativeFloatingIslandDensityAt(X,Y,Z,Density))return "FloatingIslands-native"; if(NativeSurfaceDensityAt(X,Y,Z,Density))return "SurfaceWorld-native"; if(UsesLegacySurfaceField(X,Y,Z))return "SurfaceWorld-legacy"; - if(Owner&&Owner->Manager) + const FBoundStratumField* Bound=FindBoundStratumForZ(Z); + if(Bound&&Bound->Resolved) { - const int32 ChunkZ=FMath::FloorToInt(Z/static_cast(GetResolvedLayoutChunkWidthVoxels())); - const ECaveGeneratorType Type=Owner->Manager->GetGeneratorTypeForChunk(FIntVector(0,0,ChunkZ)); - switch(Type) - { - case ECaveGeneratorType::TunnelNetwork:return "compiled-analytic/TunnelNetwork"; - case ECaveGeneratorType::FlatPlain:return "compiled-analytic/FlatPlain"; - case ECaveGeneratorType::CrystalChamber:return "compiled-analytic/CrystalChamber"; - case ECaveGeneratorType::Maze:return "compiled-analytic/Maze"; - case ECaveGeneratorType::SurfaceWorld:return "compiled-analytic/SurfaceWorld"; - case ECaveGeneratorType::VerticalShafts:return "compiled-analytic/VerticalShafts"; - case ECaveGeneratorType::FloatingIslands:return "compiled-analytic/FloatingIslands"; - case ECaveGeneratorType::Underwater:return "compiled-analytic/Underwater"; - } + const std::string& Kind=Bound->Resolved->generatorKind; + return "compiled-analytic/"+Kind; } - return "compiled-analytic/fallback"; + return Owner?"compiled-analytic/TunnelNetwork":"compiled-analytic/fallback"; } float DensityAtLattice(int32 X,int32 Y,int32 Z) const @@ -3672,7 +3883,10 @@ private: &&!NativeSurfaceDensityAt(X,Y,Z,density)) { if(UsesLegacySurfaceField(X,Y,Z)&&Owner&&Owner->Generator) - density=Owner->Generator->GetDensityAt(X,Y,Z); + { + if(!LegacyGeneratorDensityAt(X,Y,Z,density)) + density=Owner->Generator->GetDensityAt(X,Y,Z); + } else density=AnalyticDensityAt(X,Y,Z,true,nullptr,nullptr,nullptr, false,false); @@ -3961,7 +4175,9 @@ private: NativeDensity=ApplyResolvedStrateComposition(NativeDensity,X,Y,Z); return bApplyEdits?ApplyEditBrushes(NativeDensity,X,Y,Z):NativeDensity; } - const float Density=Owner->Generator->GetDensityAt(X,Y,Z); + float Density=0.0f; + if(!LegacyGeneratorDensityAt(X,Y,Z,Density)) + Density=Owner->Generator->GetDensityAt(X,Y,Z); const float composed=bApplyComposition ?ApplyResolvedStrateComposition(Density,X,Y,Z):Density; return bApplyEdits?ApplyEditBrushes(composed,X,Y,Z):composed; @@ -4019,23 +4235,19 @@ public: { if(SharedDensitySource) return SharedDensitySource->GetNativeSurfaceHeightsAt(X,Y,ChunkZ,OutGroundZ,OutCeilingZ); - if(!WorldAssembly||!ResolvedLayout||!Owner||!Owner->Manager||!Owner->Generator + if(!WorldAssembly||!ResolvedLayout||!Owner||!Owner->Generator ||!std::isfinite(X)||!std::isfinite(Y))return false; const double ChunkWidth=GetResolvedLayoutChunkWidthVoxels(); const std::int64_t chunkMinimum=static_cast(std::llround(ChunkZ*ChunkWidth)); const std::int64_t chunkMaximum=static_cast(std::llround((ChunkZ+1)*ChunkWidth)); + const FBoundStratumField* Bound=FindBoundStratumForChunk(ChunkZ); + const auto* Field=Bound?std::get_if(&Bound->Evaluator):nullptr; if(bNativeOnly&&(!NativeOnlyContainsColumn(X,Y) ||chunkMaximumNativeOnlyMaximum.Z - ||Owner->Manager->GetGeneratorTypeForChunk(FIntVector(0,0,ChunkZ)) - !=ECaveGeneratorType::SurfaceWorld)) + ||!Bound||Bound->Kind!=FBoundStratumField::EKind::SurfaceWorld)) {RecordNativeOnlyViolation();return false;} - if(Owner->Manager->GetGeneratorTypeForChunk(FIntVector(0,0,ChunkZ)) - !=ECaveGeneratorType::SurfaceWorld)return false; - const int32 StrateIndex=Owner->Manager->GetStrateIndex( - static_cast((static_cast(ChunkZ)+0.5) - *ResolvedLayout->strateLayoutChunkSizeMicrometres/10000.0)); - const auto* Plan=FindWorldAssemblyStratum(StrateIndex,ChunkZ); - if(!Plan)return false; + if(!Bound||Bound->Kind!=FBoundStratumField::EKind::SurfaceWorld + ||!Bound->SurfacePlan||!Field||!Field->Evaluator.isValid())return false; constexpr double Scale=static_cast(VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel); const double MaxVoxel=static_cast(std::numeric_limits::max())/Scale; if(std::abs(static_cast(X))>MaxVoxel||std::abs(static_cast(Y))>MaxVoxel) @@ -4043,9 +4255,7 @@ public: const auto ToFixed=[](float Value) {return static_cast(std::llround(static_cast(Value) *VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel));}; - const VoxelForgeCore::Heightfield2D Block(WorldAssembly->recipe,*Plan); - if(!Block.isValid())return false; - const auto Column=Block.point(ToFixed(X),ToFixed(Y), + const auto Column=Field->Evaluator.point(ToFixed(X),ToFixed(Y), static_cast(ChunkZ)*ResolvedLayout->strateLayoutChunkWidthSubvoxels); OutGroundZ=static_cast(Column.groundHeightSubvoxels/Scale); OutCeilingZ=static_cast(Column.capHeightSubvoxels/Scale); @@ -4058,25 +4268,22 @@ public: bool IsNativeSurfaceBox(const FIntVector& Origin,int32 Step,int32 Cells) const { if(SharedDensitySource)return SharedDensitySource->IsNativeSurfaceBox(Origin,Step,Cells); - if(!WorldAssembly||!Owner||!Owner->Manager||Step<=0||Cells<=0)return false; + if(!WorldAssembly||!ResolvedLayout||!Owner||Step<=0||Cells<=0)return false; const int32 Pad=std::max(Step,1); const int32 BMin=Origin.Z-Pad; const int32 BMax=Origin.Z+(Cells+1)*Step; const float LayoutWidth=static_cast(GetResolvedLayoutChunkWidthVoxels()); const int32 FirstChunk=FMath::FloorToInt(static_cast(BMin)/LayoutWidth); const int32 LastChunk=FMath::FloorToInt(static_cast(BMax)/LayoutWidth); - const VoxelForgeCore::WorldAssemblyStratumPlanV1* Plan=nullptr; + const FBoundStratumField* BoundPlan=nullptr; for(int32 ChunkZ=FirstChunk;ChunkZ<=LastChunk;++ChunkZ) { - const auto It=std::find_if(WorldAssembly->plan.strates.begin(), - WorldAssembly->plan.strates.end(),[ChunkZ](const auto& Entry) - {return ChunkZ<=Entry.topChunkZ&&ChunkZ>=Entry.bottomChunkZ;}); - if(It==WorldAssembly->plan.strates.end())return false; - const auto* Candidate=&*It; - if(Plan&&Plan!=Candidate)return false; - if(Owner->Manager->GetGeneratorTypeForChunk(FIntVector(0,0,ChunkZ)) - !=ECaveGeneratorType::SurfaceWorld)return false; - Plan=Candidate; + const auto* Candidate=FindBoundStratumForChunk(ChunkZ); + if(!Candidate||Candidate->Kind!=FBoundStratumField::EKind::SurfaceWorld + ||!Candidate->SurfacePlan||ChunkZSurfacePlan->bottomChunkZ + ||ChunkZ>Candidate->SurfacePlan->topChunkZ)return false; + if(BoundPlan&&BoundPlan!=Candidate)return false; + BoundPlan=Candidate; } if(bNativeOnly) { @@ -4088,8 +4295,8 @@ public: ||BMaxY>NativeOnlyMaximum.Y||BMax>NativeOnlyMaximum.Z) {RecordNativeOnlyViolation();return false;} } - if(bNativeOnly&&!Plan)RecordNativeOnlyViolation(); - return Plan!=nullptr; + if(bNativeOnly&&!BoundPlan)RecordNativeOnlyViolation(); + return BoundPlan!=nullptr; } FBoxClassification ClassifyBoxDetailed(const FIntVector& Origin,int32 Step,int32 Cells, @@ -4104,31 +4311,39 @@ public: int32 BMax[3]={Origin.X+(Cells+1)*Step,Origin.Y+(Cells+1)*Step,Origin.Z+(Cells+1)*Step}; const VoxelForgeCore::WorldAssemblyStratumPlanV1* SurfacePlan=nullptr; const VoxelForgeCore::WorldAssemblyFloatingIslandPlanV1* IslandPlan=nullptr; + const FSurfaceWorldFieldAdapter* SurfaceField=nullptr; + const FFloatingIslandFieldAdapter* IslandField=nullptr; bool bSurfaceOverlap=false,bIslandOverlap=false,bOtherStrateOverlap=false; bool bIslandOtherStrateOverlap=false,bBadPlan=false; - if(Owner&&Owner->Manager&&ResolvedLayout) + if(Owner&&ResolvedLayout) { const float LayoutWidth=static_cast(GetResolvedLayoutChunkWidthVoxels()); const int32 FirstChunk=FMath::FloorToInt(static_cast(BMin[2])/LayoutWidth); const int32 LastChunk=FMath::FloorToInt(static_cast(BMax[2])/LayoutWidth); for(int32 ChunkZ=FirstChunk;ChunkZ<=LastChunk;++ChunkZ) { - const ECaveGeneratorType Type=Owner->Manager->GetGeneratorTypeForChunk(FIntVector(0,0,ChunkZ)); - const int32 StrateIndex=Owner->Manager->GetStrateIndex(static_cast( - (static_cast(ChunkZ)+0.5)*ResolvedLayout->strateLayoutChunkSizeMicrometres/10000.0)); - if(Type==ECaveGeneratorType::SurfaceWorld) + const FBoundStratumField* Bound=FindBoundStratumForChunk(ChunkZ); + if(Bound&&Bound->Kind==FBoundStratumField::EKind::SurfaceWorld) { bSurfaceOverlap=true;bIslandOtherStrateOverlap=true; - const auto* Candidate=FindWorldAssemblyStratum(StrateIndex,ChunkZ); - if(!Candidate||(SurfacePlan&&SurfacePlan!=Candidate))bBadPlan=true; + const auto* Candidate=Bound->SurfacePlan; + if(!Candidate||ChunkZbottomChunkZ + ||ChunkZ>Candidate->topChunkZ + ||(SurfacePlan&&SurfacePlan!=Candidate))bBadPlan=true; SurfacePlan=Candidate; + SurfaceField=std::get_if( + &Bound->Evaluator); } - else if(Type==ECaveGeneratorType::FloatingIslands) + else if(Bound&&Bound->Kind==FBoundStratumField::EKind::FloatingIslands) { bIslandOverlap=true;bOtherStrateOverlap=true; - const auto* Candidate=FindWorldAssemblyFloatingIslandStratum(StrateIndex,ChunkZ); - if(!Candidate||(IslandPlan&&IslandPlan!=Candidate))bBadPlan=true; + const auto* Candidate=Bound->IslandPlan; + if(!Candidate||ChunkZbottomChunkZ + ||ChunkZ>Candidate->topChunkZ + ||(IslandPlan&&IslandPlan!=Candidate))bBadPlan=true; IslandPlan=Candidate; + IslandField=std::get_if( + &Bound->Evaluator); } else {bOtherStrateOverlap=true;bIslandOtherStrateOverlap=true;} @@ -4213,12 +4428,12 @@ public: } if(bSurfaceOverlap) { - if(!WorldAssembly||!SurfacePlan)return Finish(EVoxelTileClass::Mixed,true); - const VoxelForgeCore::Heightfield2D Block(WorldAssembly->recipe,*SurfacePlan); + if(!WorldAssembly||!SurfacePlan||!SurfaceField) + return Finish(EVoxelTileClass::Mixed,true); // classifyLatticeBox takes the exact extrema of ground/cap columns at every // MC lattice XY in this box. Since density is min(z-ground, cap-z), those extrema // bound every lattice sample in Z without a noise amplitude guess. - const auto Verdict=Block.classifyLatticeBox( + const auto Verdict=SurfaceField->Evaluator.classifyLatticeBox( static_cast(BMin[0])*VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel, static_cast(BMin[1])*VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel, static_cast(BMin[2])*VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel, @@ -4239,7 +4454,7 @@ public: } if(bIslandOverlap) { - if(!IslandPlan)return Finish(EVoxelTileClass::Mixed,true); + if(!IslandPlan||!IslandField)return Finish(EVoxelTileClass::Mixed,true); constexpr std::int64_t Scale=VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel; const std::int64_t Bottom=IslandPlan->manifest.recipe.layoutMinimumZSubvoxels; const std::int64_t Top=IslandPlan->manifest.recipe.layoutMaximumZSubvoxels; @@ -4259,10 +4474,8 @@ public: return Finish((IslandPlan->manifest.recipe.geometryVersion<3 ?BoxBottom>=Top-Seal:BoxBottom>Top-Seal)&&BoxTop<=Top ?EVoxelTileClass::AllSolid:EVoxelTileClass::Mixed); - const VoxelForgeCore::FloatingIslandFieldV1 Block( - IslandPlan->manifest.recipe,IslandPlan->manifest.plan); - if(!Block.isValid())return Finish(EVoxelTileClass::Mixed,true); - const auto Interval=Block.interval(static_cast(BMin[0])*Scale, + if(!IslandField->Evaluator.isValid())return Finish(EVoxelTileClass::Mixed,true); + const auto Interval=IslandField->Evaluator.interval(static_cast(BMin[0])*Scale, static_cast(BMin[1])*Scale,static_cast(BMin[2])*Scale, static_cast(BMax[0])*Scale,static_cast(BMax[1])*Scale, static_cast(BMax[2])*Scale); @@ -4782,30 +4995,18 @@ private: bool NativeSurfaceDensityAt(float X,float Y,float Z,float& OutDensity) const { - if(!bUseNativeManifestFields||!WorldAssembly||!ResolvedLayout||!Owner||!Owner->Manager||!Owner->Generator - ||!std::isfinite(X)||!std::isfinite(Y)||!std::isfinite(Z))return false; + if(!bUseNativeManifestFields||!WorldAssembly||!ResolvedLayout||!Owner + ||!Owner->Manager||!Owner->Generator||!std::isfinite(X) + ||!std::isfinite(Y)||!std::isfinite(Z))return false; const float LayoutWidth=static_cast(GetResolvedLayoutChunkWidthVoxels()); if(!(LayoutWidth>0.0f))return false; const int32 ChunkZ=FMath::FloorToInt(Z/LayoutWidth); - if(Owner->Manager->GetGeneratorTypeForChunk(FIntVector(0,0,ChunkZ)) - !=ECaveGeneratorType::SurfaceWorld)return false; - const int32 StrateIndex=Owner->Manager->GetStrateIndex( - static_cast((static_cast(ChunkZ)+0.5) - *ResolvedLayout->strateLayoutChunkSizeMicrometres/10000.0)); - const auto* Plan=FindWorldAssemblyStratum(StrateIndex,ChunkZ); - if(!Plan)return false; - constexpr double Scale=static_cast(VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel); - const double MaxVoxel=static_cast(std::numeric_limits::max())/Scale; - if(std::abs(static_cast(X))>MaxVoxel||std::abs(static_cast(Y))>MaxVoxel - ||std::abs(static_cast(Z))>MaxVoxel)return false; - const auto ToFixed=[](float Value) - {return static_cast(std::llround(static_cast(Value) - *VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel));}; - const VoxelForgeCore::Heightfield2D Block(WorldAssembly->recipe,*Plan); - if(!Block.isValid())return false; - OutDensity=static_cast(Block.point(ToFixed(X),ToFixed(Y),ToFixed(Z)).densitySubvoxels - /Scale); - return true; + const FBoundStratumField* Bound=FindBoundStratumForChunk(ChunkZ); + if(!Bound||Bound->Kind!=FBoundStratumField::EKind::SurfaceWorld + ||!Bound->SurfacePlan||ChunkZSurfacePlan->bottomChunkZ + ||ChunkZ>Bound->SurfacePlan->topChunkZ)return false; + const auto* Field=std::get_if(&Bound->Evaluator); + return Field&&Field->densityAt(X,Y,Z,OutDensity); } bool NativeFloatingIslandDensityAt(float X,float Y,float Z,float& OutDensity) const @@ -4818,13 +5019,15 @@ private: const double LayoutChunkWidth=GetResolvedLayoutChunkWidthVoxels(); if(!(LayoutChunkWidth>0.0))return false; const int32 ChunkZ=FMath::FloorToInt(Z/static_cast(LayoutChunkWidth)); - if(Owner->Manager->GetGeneratorTypeForChunk(FIntVector(0,0,ChunkZ)) - !=ECaveGeneratorType::FloatingIslands)return false; - const int32 StrateIndex=Owner->Manager->GetStrateIndex( - static_cast((static_cast(ChunkZ)+0.5) - *ResolvedLayout->strateLayoutChunkSizeMicrometres/10000.0)); - const auto* Plan=FindWorldAssemblyFloatingIslandStratum(StrateIndex,ChunkZ); - if(!Plan)return false; + const FBoundStratumField* Bound=FindBoundStratumForChunk(ChunkZ); + if(!Bound||Bound->Kind!=FBoundStratumField::EKind::FloatingIslands + ||!Bound->IslandPlan||ChunkZIslandPlan->bottomChunkZ + ||ChunkZ>Bound->IslandPlan->topChunkZ)return false; + const int32 StrateIndex=Bound->StrateIndex; + const auto* Plan=Bound->IslandPlan; + const auto* FieldAdapter=std::get_if( + &Bound->Evaluator); + if(!FieldAdapter||!FieldAdapter->Evaluator.isValid())return false; constexpr double Scale=static_cast(VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel); constexpr double Maximum=1000000000000.0/Scale; if(std::abs(static_cast(X))>Maximum||std::abs(static_cast(Y))>Maximum @@ -4832,11 +5035,8 @@ private: const auto ToFixed=[](float Value) {return static_cast(std::llround(static_cast(Value) *VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel));}; - const VoxelForgeCore::FloatingIslandFieldV1 Field( - Plan->manifest.recipe,Plan->manifest.plan); - if(!Field.isValid())return false; const std::int64_t qx=ToFixed(X),qy=ToFixed(Y),qz=ToFixed(Z); - std::int64_t Density=Field.densitySubvoxels(qx,qy,qz); + std::int64_t Density=FieldAdapter->densitySubvoxelsAt(qx,qy,qz); const std::int64_t NativeDensity=Density; // The anchor-root field keeps broad lobes, but its shortest authored noise // wavelength is below one metre. Filter that detail only around a FloatingIslands @@ -4963,7 +5163,7 @@ private: { const std::int64_t weight=(dx==0?2:1)*(dy==0?2:1)*(dz==0?2:1); const std::int64_t sample=(dx==0&&dy==0&&dz==0)?Density: - Field.densitySubvoxels(qx+dx*FilterStep,qy+dy*FilterStep, + FieldAdapter->densitySubvoxelsAt(qx+dx*FilterStep,qy+dy*FilterStep, qz+dz*FilterStep); weightedSum+=sample*weight; } @@ -5000,7 +5200,7 @@ private: { std::array sample{qx,qy,qz}; sample[axis]+=sign*fragmentStep; - const std::int64_t neighbor=Field.densitySubvoxels( + const std::int64_t neighbor=FieldAdapter->densitySubvoxelsAt( sample[0],sample[1],sample[2]); if(neighbor<=0) { @@ -5031,11 +5231,11 @@ private: if(!MakeResolvedStratumBounds(static_cast(Bottom)/Scale, static_cast(Top)/Scale,static_cast(Seal)/Scale,Bounds)) return false; - const FFloatingIslandParams Params=Owner->Manager->GetFloatingIslandParamsForChunk( - FIntVector(0,0,ChunkZ)); + const FFloatingIslandParams* Params=FieldAdapter->legacyBoundaryParamsAt(ChunkZ); + if(!Params)return false; float InternalDensity=-OutDensity; VF_ApplyUpperBoundarySeal(InternalDensity,Z,static_cast(Bounds.TopZ), - static_cast(Bounds.SealThickness),Params.BaseDensity); + static_cast(Bounds.SealThickness),Params->BaseDensity); OutDensity=-InternalDensity; } else OutDensity=static_cast(static_cast(Density)/Scale); @@ -5044,12 +5244,16 @@ private: bool UsesLegacySurfaceField(float X,float Y,float Z) const { - if(!Owner||!Owner->Manager||!Owner->Generator)return false; - const double LayoutChunkWidth=GetResolvedLayoutChunkWidthVoxels(); - if(!(LayoutChunkWidth>0.0))return false; - const FIntVector ChunkCoord(0,0,FMath::FloorToInt(Z/static_cast(LayoutChunkWidth))); - return Owner->Manager->GetGeneratorTypeForChunk(ChunkCoord) - ==ECaveGeneratorType::SurfaceWorld; + const FBoundStratumField* Bound=FindBoundStratumForZ(Z); + return Bound&&Bound->Kind==FBoundStratumField::EKind::SurfaceWorld; + } + + bool LegacyGeneratorDensityAt(float X,float Y,float Z,float& OutDensity) const + { + const FBoundStratumField* Bound=FindBoundStratumForZ(Z); + const auto* Legacy=Bound?std::get_if( + &Bound->Evaluator):nullptr; + return Legacy&&Legacy->densityAt(X,Y,Z,OutDensity); } float LegacySurfaceDensityAt(float X,float Y,float Z) const @@ -5057,7 +5261,10 @@ private: // SurfaceWorld consumes biome overrides, terrain ops, water level and its sky cap in // the legacy stack. Reuse that immutable evaluator as an exact temporary bridge. // SurfaceWorld lit biomes, ops de terrain, niveau d'eau et plafond; reutiliser ce champ. - const float density=ApplyResolvedStrateComposition(Owner->Generator->GetDensityAt(X,Y,Z),X,Y,Z); + float density=0.0f; + if(!LegacyGeneratorDensityAt(X,Y,Z,density)) + density=Owner->Generator->GetDensityAt(X,Y,Z); + density=ApplyResolvedStrateComposition(density,X,Y,Z); return ApplyEditBrushes(density,X,Y,Z); } @@ -8378,11 +8585,13 @@ private: bool Built=false; bool bNativeOnly=false; bool bUseNativeManifestFields=false; + float BoundManagerLayoutChunkWidthVoxels=0.0f; FIntVector NativeOnlyMinimum=FIntVector::ZeroValue; FIntVector NativeOnlyMaximum=FIntVector::ZeroValue; mutable std::atomic NativeOnlyViolations{0}; bool FarBuilt=false; std::vector> ResolvedBoundaryZIntervalsMicrometres; + std::vector BoundStratumFields; std::vector LayerRanges; std::vector LayerFields; std::vector Primitives; diff --git a/Standalone/Research/WorldAssemblerM3/Linux40/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux40/REPORT.md new file mode 100644 index 0000000..9d9f7b6 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux40/REPORT.md @@ -0,0 +1,74 @@ +# Linux40 — immutable field binding + +## Change + +`ShapeWorldInternal.h` now has concrete immutable SurfaceWorld, FloatingIslands and legacy adapters +with a common `densityAt(x,y,z,out)` call shape. `FGeneratorFieldAdapterV1` stores them in a variant, +so hot sampling uses concrete dispatch without a virtual call. FI retains its fixed-point sampler and +captures the legacy boundary parameters while binding. + +`BindResolvedStratumFields` creates a sorted table of resolved strate bounds, plan references and +evaluator objects during world construction/initialization. Native SurfaceWorld and FI sampling, +boundary composition lookup, legacy SurfaceWorld bridging and `ClassifyBoxDetailed` select from that +table. The manager layout width is captured once for the legacy-compatible chunk projection. + +## Lookup boundary and identity + +The direct manager selection lookups in native SurfaceWorld/FI evaluation and box classification are +removed. `DensityAt` now selects native fields through the bound table. Legacy +`UVoxelGenerator::GetDensityAt` still performs its own manager routing when reached through the +legacy bridge or an out-of-plan fallback. Manager-derived layout width/FI compatibility parameters +are captured at bind time; manager access also remains in build/planning helpers. Generator-specific +boundary and passage policy still lives in `FStandaloneShapeWorld`. + +No density arithmetic, fixed-point conversion order, evaluator version or Unreal adapter changed. +The canonical owner FI and SurfaceWorld hashes are identical to L37/L39 for W1 and W12. The owner +layout IDs also match L39: FI `b7a7d9ecc48d8f26`, SurfaceWorld `8a52e84c26ed92d2`. Fast logs and hash +summaries: `Saved/Linux/l40/fast-trial1.log` and `Saved/Linux/l40/fast-summary.json`. + +## Cost + +Five interleaved pairs against the L39 Clang binary, using the canonical owner recipes: + +| Recipe | L39 median | L40 median | Median change | Paired changes (L40 vs L39) | +|---|---:|---:|---:|---| +| FloatingIslands | 17.556 s | 17.440 s | −0.66% | +2.26%, −3.99%, −0.66%, −0.87%, −0.47% | +| SurfaceWorld | 1.829 s | 1.805 s | −1.31% | −2.64%, −0.22%, +0.55%, −0.44%, −1.48% | + +Both deltas are within observed pair noise; no per-voxel regression was measured. Raw output is in +`Saved/Linux/l40/cost-l40-vs-l39-run.log` and `cost-l40-vs-l39-summary.json`. + +## Verification + +| Tier | Result | Wall time | Evidence | +|---|---|---:|---| +| Fast | PASS | 162.0 s | Canonical owner hashes match L37 for both recipes at W1/W12; all controls pass. | +| Medium | PASS | 468.6 s | Transition, route/proof, LOD0–8 ring, owner shape and capture matrix pass. | +| Full | TIMED OUT | 1740 s (29 min) | GCC build and supplement passed; L25 and crossing24 determinism passed for both compilers, and Clang upper-SW style0 W1/W8/W12 passed. The run timed out during the GCC upper-SW style0 W1 cell; no full summary or cost-baseline/pair results were produced. | + +Clang and GCC candidate builds succeeded with source stamp +`f35bc59c6172370e98dc83ec8ee5ba51dfab83a3faf075c26189fc83172f4ac8`. The full run used the requested +CPU/memory limits and a 1740-second timeout. At the stop, host load was 20.5. The partial log is +`Saved/Linux/l40/full-final.log`; a `full-summary.json` was not written. Other tier and build logs are +under `Saved/Linux/l40/`. + +## Failed attempts + +- The first Clang build passed `unique_ptr` objects to adapter constructors expecting raw observer + pointers; corrected with `.get()`, then the build passed. +- Two early cost-harness launches failed before starting measurements: one shell quoting error and + two Python `-c` quoting errors. The later logged five-pair run completed successfully. +- The single full-tier run reached its 1740-second timeout while the GCC upper-SW fixture style0 W1 + determinism cell was still running. The cell has no completion record; the run returned timeout + status 124. No full-tier L36 baseline build or full-tier cost pairs ran. + +No Unreal compile was run; no Unreal adapter or `VoxelWorld.cpp` source change was required. + +## Orchestrator review (Claude) + +Accepted. The full tier had timed out here under load (two Lunas plus host load of about 20). I re-ran it alone on +the final source (stamp `f35bc59c…`): **PASS, 407 gates, 2857 s.** Its built-in cost comparison is against +**L36** and shows FI +3.03% (all five pairs positive) and SW +0.38%. That FI delta is L37's mouth geometry; this +round's own L39→L40 pairs are −0.66% FI and −1.31% SW. Two follow-ups for the tool: +- the full tier's cost baseline is hard-coded to L36; it should follow the previous commit; +- the full tier now needs about 48 minutes including builds, so its timeout must be above 29 minutes. diff --git a/Standalone/Research/WorldAssemblerM3/Research7/REPORT.md b/Standalone/Research/WorldAssemblerM3/Research7/REPORT.md new file mode 100644 index 0000000..192939a --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Research7/REPORT.md @@ -0,0 +1,45 @@ +# Research 7 — terrain disk cache: Windows and multiplayer + +Inspection date: 2026-10-23. Read-only review of `Standalone/Source/StreamingWorld.h`, `Standalone/Source/StandaloneViewer.cpp`, Core version declarations, and the verification README. + +## What exists and Windows blockers + +`FStandaloneMeshCache` is a whole-world archive, not an on-demand disk cache. Its filename is derived from seed, clip radius, and tunnel mode; the header also checks source JSON FNV-1a-64, generation settings, cache/algorithm version, tunnel mode, and voxel size. A tile lookup uses `(LOD, X, Y, Z)`. The `VFMESH1\0` v1 archive has an 80-byte raw C++ header, 40-byte raw tile records, float32 position/normal pairs, and uint32 indices. It is uncompressed and host-layout/endian dependent; the endian marker detects mismatch but does not convert fields. Save/load also omit generated `SurfaceVertexCount`, `SkirtTriangleCount`, and `TriangleSkirtFaceMasks`, so a cache hit cannot preserve skirt culling. + +Load reads and validates the entire archive synchronously in `ConfigureProfile` before workers start. Streaming workers then copy hits from an immutable in-memory map. Bake save writes synchronously to the final path with truncation. There is no disk eviction or file lock; leaving tiles evicts resident meshes only. The cache has no `mmap`, `fsync`, rename, or zlib use today. Existing checks reject mismatched metadata, invalid LOD/counts, truncated data, duplicate keys, non-finite positions/normals, and out-of-range indices, but there is no payload checksum, total allocation budget, or trailing-data check. Stored density hashes are not checked against the mesh. + +Windows concerns are the narrow `.string()` path conversion and a fixed `MAX_PATH` buffer in `DefaultDataPath`; the cache is written next to source data, which may be read-only in a packaged install. Windows long-path support needs application opt-in, and the cache path should stay in TCHAR/UE paths. Direct-to-final truncation can leave a partial archive and destroy the previous one; `ofstream::flush` is not a durable or atomic replacement. Concurrent bakes can collide. The standalone viewer itself has unguarded Win32/D3D11 headers and should not be pulled into the UE plugin; keep any platform adapter gated and use UE file APIs. + +## UE5 design + +- Store a versioned, endian-neutral per-tile payload under `FPaths::ProjectSavedDir()/VoxelForge/TileCache///`. Use `ProjectPersistentDownloadDir()` when the packaged install’s Saved directory is not writable; append a project identity either way. [FPaths API](https://dev.epicgames.com/documentation/en-us/unreal-engine/API/Runtime/Core/FPaths) +- Use `FPlatformFileManager::Get().GetPlatformFile().OpenAsyncRead()` and `IAsyncReadFileHandle::ReadRequest()` for tile/block reads. Parse and decompress on a worker; never wait for file I/O or decompression on the game thread. UE documents async open as non-blocking and read requests as caller-owned; keep buffers/requests alive through completion and cancellation. [FPlatformFileManager](https://dev.epicgames.com/documentation/unreal-engine/API/Runtime/Core/FPlatformFileManager), [IPlatformFile](https://dev.epicgames.com/documentation/unreal-engine/API/Runtime/Core/IPlatformFile), [ReadRequest](https://dev.epicgames.com/documentation/unreal-engine/API/Runtime/Core/Async/IAsyncReadFileHandle/ReadRequest?application_version=5.5) +- Use UE `FCompression` with registered format names and `IsFormatValid`; its compression/decompression routines are thread-safe. This avoids a separately linked Windows zlib. Keep codec ID/version in each tile record and check availability in the packaged client/server. Oodle Data is supplied as a UE plugin and documents Kraken, Mermaid, and Selkie as distinct size/decode-speed choices; do not assume a packaging setting alone enables a runtime cache codec. [FCompression](https://dev.epicgames.com/documentation/en-us/unreal-engine/API/Runtime/Core/FCompression), [Oodle Data (UE 5.7)](https://dev.epicgames.com/documentation/en-us/unreal-engine/oodle-data?application_version=5.7) +- Treat the generated terrain as persisted feature data, not as a speed claim. Record schema, world key, tile key, codec, compressed/uncompressed sizes, and payload digest. Include all mesh groups, masks, and collision inputs required to rebuild identical tiles. Write a unique temp beside the target, flush and close it, then replace/rename on the same volume. Do not allow a cross-volume copy fallback. Use `ReplaceFileW`/same-volume `MoveFileExW` through a Windows adapter; `FlushFileBuffers` is the file-data durability step. [ReplaceFileW](https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-replacefilew), [MoveFileExW](https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw), [FlushFileBuffers](https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-flushfilebuffers), [Windows long paths](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation) + +Cache misses, corrupt entries, and unavailable codecs fall back to cold generation on workers. Keep eviction as a background size-bounded LRU; cache eviction only causes regeneration. Continue judging the no-cache cold path separately. + +## Key, multiplayer, invalidation, validation + +Key a base tile by authoritative season/world-plan identity (include Core recipe and plan content hashes), recipe/schema version, evaluator version, selected mesher plus an explicit mesher implementation revision, LOD and absolute tile coordinate. Add the development source stamp and cache-format version. Core currently declares WorldAssembly schema 2/evaluator 208; FloatingIsland recipe schema 1/default geometry version 4; mesher choices are MarchingCubesV1, HybridV1, HybridV2, with far-horizon version 4 affecting LOD4–8. Read actual resolved recipe/plan versions and hashes from the active snapshot. The verification README defines a SHA-256 source stamp over a sorted source manifest; it is dynamic and can optionally include the Unreal adapter. The current cache uses only the source-data FNV hash and does not carry that stamp. + +Persist immutable base terrain only. Keep player edits in the replicated diff layer and overlay them at runtime. Do not write edited composite tiles into the base cache; if a separate composite cache is ever added, key it by the affected tile’s edit revision/content hash. A client cache is visual convenience only: the server uses its own matching cache or regenerates and builds its own collision; never accept client cache bytes as collision or gameplay authority. + +On load, bound both per-record and total tile/byte allocations before allocating; require exact file/block lengths, known codec/schema/key, finite and bounded geometry, valid normals, triangle/index/mask counts, and index ranges. Verify a per-payload digest and regenerate on failure. A digest detects accidental corruption, not a malicious rewrite that also changes the digest; only importable/shared caches need a server signature or MAC. Never trust a cache supplied by a client. + +## Compression probe and next build round + +A small serial Python probe used `VFMSH001` exports `linux27-fi-anchor-pillar.vfmesh` and `linux27-sw-column-wide.vfmesh`, all non-empty tiles LOD0–8, CPython zlib level 1. Payload excludes scene/tile headers. These are Linux in-memory decode numbers, not UE/MSVC or disk-I/O results; exports lack recipe/source stamps and density samples. + +| Export | Non-empty tiles | Mean raw → zlib bytes/tile | Payload ratio | Decode ms/tile p50 / p95 | +|---|---:|---:|---:|---:| +| FI | 1,380 | 75.6 KB → 24.2 KB | 32.0% | 0.109 / 0.523 | +| SW | 1,416 | 60.0 KB → 21.5 KB | 35.8% | 0.114 / 0.459 | + +The serial probe peak RSS was 15.4 MiB. Per-LOD details are in `Saved/Linux/research/R7_python_zlib_probe.json`; script: `Saved/Linux/research/R7_python_zlib_probe.py`. + +Later compare none, zlib, Oodle Kraken/Mermaid/Selkie, and LZ4; quantized vertices (uint16 positions, compact normals) with delta indices; and density/sign-bit packing with RLE. Use the same owner FI and SW tile sets at LOD0–8, keyed to the exact active recipe/plan. Record bytes/tile (median/p95/total), single-core decode ms/tile (median/p95), peak working memory, and cold disk-read-plus-decode latency separately. Validate geometry error, seams, and server collision against uncompressed output. Sign-bit inputs are absent from these mesh exports, so that candidate needs a later density export and must include remesh time in its comparison. + +## Owner question + +Will cache files remain local to each user/install, or be copied/shared between machines? diff --git a/Standalone/Research/WorldAssemblerM3/Research7/scripts/R7_python_zlib_probe.json b/Standalone/Research/WorldAssemblerM3/Research7/scripts/R7_python_zlib_probe.json new file mode 100644 index 0000000..ae0a8c8 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Research7/scripts/R7_python_zlib_probe.json @@ -0,0 +1,255 @@ +{ + "codec": "CPython zlib level 1", + "decode_repeats_per_tile": 3, + "files": { + "FI": { + "path": "Saved/Linux/linux27-fi-anchor-pillar.vfmesh", + "file_bytes": 104373453, + "scene_version": 1, + "voxel_size_microns": 250000, + "tile_records": 2482, + "nonempty_tile_records": 1380, + "raw_payload_bytes": 104284041, + "zlib_payload_bytes": 33400206, + "weighted_ratio_pct": 32.02810869210563, + "raw_bytes_per_nonempty_tile_median": 46104.0, + "decode_ms_per_tile_median": 0.109374, + "decode_ms_per_tile_p95": 0.523225, + "lod": { + "0": { + "nonempty_tiles": 136, + "raw_bytes_total": 11914151, + "packed_bytes_total": 3218452, + "weighted_ratio_pct": 27.01369153370643, + "raw_bytes_median": 95849.5, + "packed_bytes_median": 33744.0, + "decode_ms_median": 0.27110500000000004, + "decode_ms_p95": 0.316059, + "encode_ms_median": 0.610732, + "encode_ms_p95": 0.837242 + }, + "1": { + "nonempty_tiles": 131, + "raw_bytes_total": 16499945, + "packed_bytes_total": 5941348, + "weighted_ratio_pct": 36.008289724602115, + "raw_bytes_median": 92598, + "packed_bytes_median": 42032, + "decode_ms_median": 0.317582, + "decode_ms_p95": 0.818547, + "encode_ms_median": 0.837591, + "encode_ms_p95": 2.246851 + }, + "2": { + "nonempty_tiles": 172, + "raw_bytes_total": 6666423, + "packed_bytes_total": 2031078, + "weighted_ratio_pct": 30.467283579214822, + "raw_bytes_median": 45768.0, + "packed_bytes_median": 11943.0, + "decode_ms_median": 0.102419, + "decode_ms_p95": 0.124572, + "encode_ms_median": 0.2224085, + "encode_ms_p95": 0.256617 + }, + "3": { + "nonempty_tiles": 125, + "raw_bytes_total": 5555529, + "packed_bytes_total": 1649468, + "weighted_ratio_pct": 29.6905659209051, + "raw_bytes_median": 45768, + "packed_bytes_median": 11960, + "decode_ms_median": 0.104118, + "decode_ms_p95": 0.168216, + "encode_ms_median": 0.2313, + "encode_ms_p95": 0.349597 + }, + "4": { + "nonempty_tiles": 174, + "raw_bytes_total": 9645483, + "packed_bytes_total": 3115498, + "weighted_ratio_pct": 32.30007247952228, + "raw_bytes_median": 45768.0, + "packed_bytes_median": 13419.0, + "decode_ms_median": 0.1148265, + "decode_ms_p95": 0.259002, + "encode_ms_median": 0.255053, + "encode_ms_p95": 0.641356 + }, + "5": { + "nonempty_tiles": 144, + "raw_bytes_total": 4232303, + "packed_bytes_total": 1383894, + "weighted_ratio_pct": 32.69836776809222, + "raw_bytes_median": 24938.0, + "packed_bytes_median": 9315.0, + "decode_ms_median": 0.06914100000000001, + "decode_ms_p95": 0.178538, + "encode_ms_median": 0.1752125, + "encode_ms_p95": 0.460837 + }, + "6": { + "nonempty_tiles": 96, + "raw_bytes_total": 5255938, + "packed_bytes_total": 1770019, + "weighted_ratio_pct": 33.67655782849798, + "raw_bytes_median": 38395.0, + "packed_bytes_median": 10093.5, + "decode_ms_median": 0.088979, + "decode_ms_p95": 0.317363, + "encode_ms_median": 0.1975715, + "encode_ms_p95": 0.823547 + }, + "7": { + "nonempty_tiles": 96, + "raw_bytes_total": 11085771, + "packed_bytes_total": 3715174, + "weighted_ratio_pct": 33.51299607397627, + "raw_bytes_median": 105285.5, + "packed_bytes_median": 30236.0, + "decode_ms_median": 0.271746, + "decode_ms_p95": 0.588424, + "encode_ms_median": 0.63578, + "encode_ms_p95": 1.607345 + }, + "8": { + "nonempty_tiles": 306, + "raw_bytes_total": 33428498, + "packed_bytes_total": 10575275, + "weighted_ratio_pct": 31.635507524149006, + "raw_bytes_median": 100540.0, + "packed_bytes_median": 29556.0, + "decode_ms_median": 0.258287, + "decode_ms_p95": 0.529198, + "encode_ms_median": 0.644282, + "encode_ms_p95": 1.330977 + } + } + }, + "SW": { + "path": "Saved/Linux/linux27-sw-column-wide.vfmesh", + "file_bytes": 85051310, + "scene_version": 1, + "voxel_size_microns": 250000, + "tile_records": 2130, + "nonempty_tile_records": 1416, + "raw_payload_bytes": 84974570, + "zlib_payload_bytes": 30390177, + "weighted_ratio_pct": 35.76384911391726, + "raw_bytes_per_nonempty_tile_median": 38620.5, + "decode_ms_per_tile_median": 0.11431150000000001, + "decode_ms_per_tile_p95": 0.459004, + "lod": { + "0": { + "nonempty_tiles": 285, + "raw_bytes_total": 22447321, + "packed_bytes_total": 9051898, + "weighted_ratio_pct": 40.32507041708897, + "raw_bytes_median": 79066, + "packed_bytes_median": 32524, + "decode_ms_median": 0.244825, + "decode_ms_p95": 0.389183, + "encode_ms_median": 0.65028, + "encode_ms_p95": 1.155402 + }, + "1": { + "nonempty_tiles": 149, + "raw_bytes_total": 14395111, + "packed_bytes_total": 5813866, + "weighted_ratio_pct": 40.38778165725849, + "raw_bytes_median": 108155, + "packed_bytes_median": 46017, + "decode_ms_median": 0.348557, + "decode_ms_p95": 0.551613, + "encode_ms_median": 0.930073, + "encode_ms_p95": 1.662883 + }, + "2": { + "nonempty_tiles": 217, + "raw_bytes_total": 6043511, + "packed_bytes_total": 1736514, + "weighted_ratio_pct": 28.733529234909973, + "raw_bytes_median": 27184, + "packed_bytes_median": 6351, + "decode_ms_median": 0.058163, + "decode_ms_p95": 0.14959, + "encode_ms_median": 0.14067, + "encode_ms_p95": 0.37157 + }, + "3": { + "nonempty_tiles": 190, + "raw_bytes_total": 6058525, + "packed_bytes_total": 1902018, + "weighted_ratio_pct": 31.394076941169676, + "raw_bytes_median": 35120.0, + "packed_bytes_median": 7724.5, + "decode_ms_median": 0.0678105, + "decode_ms_p95": 0.15743, + "encode_ms_median": 0.17532, + "encode_ms_p95": 0.392782 + }, + "4": { + "nonempty_tiles": 173, + "raw_bytes_total": 8837946, + "packed_bytes_total": 2824214, + "weighted_ratio_pct": 31.95554713730996, + "raw_bytes_median": 35120, + "packed_bytes_median": 13578, + "decode_ms_median": 0.106095, + "decode_ms_p95": 0.267514, + "encode_ms_median": 0.252394, + "encode_ms_p95": 0.663049 + }, + "5": { + "nonempty_tiles": 144, + "raw_bytes_total": 6093522, + "packed_bytes_total": 1982234, + "weighted_ratio_pct": 32.530185334524106, + "raw_bytes_median": 35013.0, + "packed_bytes_median": 10745.0, + "decode_ms_median": 0.08670549999999999, + "decode_ms_p95": 0.186559, + "encode_ms_median": 0.230201, + "encode_ms_p95": 0.474221 + }, + "6": { + "nonempty_tiles": 96, + "raw_bytes_total": 6363816, + "packed_bytes_total": 2057514, + "weighted_ratio_pct": 32.33145018649188, + "raw_bytes_median": 61690.0, + "packed_bytes_median": 18087.0, + "decode_ms_median": 0.1538235, + "decode_ms_p95": 0.308712, + "encode_ms_median": 0.394647, + "encode_ms_p95": 0.820557 + }, + "7": { + "nonempty_tiles": 96, + "raw_bytes_total": 9716690, + "packed_bytes_total": 3399953, + "weighted_ratio_pct": 34.99085593962553, + "raw_bytes_median": 98110.0, + "packed_bytes_median": 30712.0, + "decode_ms_median": 0.2564655, + "decode_ms_p95": 0.53338, + "encode_ms_median": 0.6491285, + "encode_ms_p95": 1.50398 + }, + "8": { + "nonempty_tiles": 66, + "raw_bytes_total": 5018128, + "packed_bytes_total": 1621966, + "weighted_ratio_pct": 32.322132875048226, + "raw_bytes_median": 74425.0, + "packed_bytes_median": 23710.5, + "decode_ms_median": 0.187504, + "decode_ms_p95": 0.337573, + "encode_ms_median": 0.47401950000000004, + "encode_ms_p95": 0.885023 + } + } + } + }, + "process_peak_rss_kib": 15772 +} diff --git a/Standalone/Research/WorldAssemblerM3/Research7/scripts/R7_python_zlib_probe.py b/Standalone/Research/WorldAssemblerM3/Research7/scripts/R7_python_zlib_probe.py new file mode 100644 index 0000000..138ce58 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Research7/scripts/R7_python_zlib_probe.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Cheap per-tile zlib probe over existing VFMSH001 scenes; read-only inputs.""" +import collections +import json +import os +import resource +import statistics +import struct +import sys +import time +import zlib +from pathlib import Path + +FILES = { + "FI": Path("Saved/Linux/linux27-fi-anchor-pillar.vfmesh"), + "SW": Path("Saved/Linux/linux27-sw-column-wide.vfmesh"), +} +REPEATS = 3 + +def percentile(values, fraction): + ordered = sorted(values) + if not ordered: + return None + return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * fraction))] + +def scan(path): + rows = collections.defaultdict(list) + with path.open("rb") as stream: + prefix = stream.read(60) + if len(prefix) != 60 or prefix[:8] != b"VFMSH001": + raise RuntimeError(f"{path}: not a VFMSH001 scene") + version, voxel_um, tile_count = struct.unpack_from("<3I", prefix, 8) + if version != 1 or tile_count > 1_000_000: + raise RuntimeError(f"{path}: unsupported header") + for _ in range(tile_count): + header = stream.read(36) + if len(header) != 36: + raise EOFError(f"{path}: truncated record") + key = struct.unpack_from("<4i", header) + _, _, vertex_count, index_count, mask_count = struct.unpack_from("<5I", header, 16) + if key[3] < 0 or key[3] > 8 or index_count % 3: + raise RuntimeError(f"{path}: malformed tile record") + raw_size = vertex_count * 24 + index_count * 4 + mask_count + payload = stream.read(raw_size) + if len(payload) != raw_size: + raise EOFError(f"{path}: truncated tile payload") + if raw_size == 0: + continue + encode_start = time.perf_counter_ns() + packed = zlib.compress(payload, level=1) + encode_ms = (time.perf_counter_ns() - encode_start) / 1_000_000.0 + decode_ms = [] + for _ in range(REPEATS): + start = time.perf_counter_ns() + decoded = zlib.decompress(packed) + decode_ms.append((time.perf_counter_ns() - start) / 1_000_000.0) + if decoded != payload: + raise RuntimeError(f"{path}: round-trip mismatch") + rows[key[3]].append({ + "raw": raw_size, "packed": len(packed), + "encode_ms": encode_ms, "decode_ms": statistics.median(decode_ms), + "vertices": vertex_count, "indices": index_count, + }) + if stream.read(1): + raise RuntimeError(f"{path}: trailing data") + return version, voxel_um, tile_count, rows + +def main(): + if hasattr(os, "sched_getaffinity"): + os.sched_setaffinity(0, {min(os.sched_getaffinity(0))}) + result = {"codec": "CPython zlib level 1", "decode_repeats_per_tile": REPEATS, "files": {}} + for name, path in FILES.items(): + version, voxel_um, tile_count, by_lod = scan(path) + lod_result = {} + for lod, rows in sorted(by_lod.items()): + raw = [r["raw"] for r in rows] + packed = [r["packed"] for r in rows] + dec = [r["decode_ms"] for r in rows] + enc = [r["encode_ms"] for r in rows] + lod_result[str(lod)] = { + "nonempty_tiles": len(rows), + "raw_bytes_total": sum(raw), + "packed_bytes_total": sum(packed), + "weighted_ratio_pct": 100.0 * sum(packed) / sum(raw), + "raw_bytes_median": statistics.median(raw), + "packed_bytes_median": statistics.median(packed), + "decode_ms_median": statistics.median(dec), + "decode_ms_p95": percentile(dec, 0.95), + "encode_ms_median": statistics.median(enc), + "encode_ms_p95": percentile(enc, 0.95), + } + total_raw = sum(x["raw_bytes_total"] for x in lod_result.values()) + total_packed = sum(x["packed_bytes_total"] for x in lod_result.values()) + all_decode = [r["decode_ms"] for rows in by_lod.values() for r in rows] + all_raw = [r["raw"] for rows in by_lod.values() for r in rows] + result["files"][name] = { + "path": str(path), "file_bytes": path.stat().st_size, + "scene_version": version, "voxel_size_microns": voxel_um, + "tile_records": tile_count, "nonempty_tile_records": sum(len(v) for v in by_lod.values()), + "raw_payload_bytes": total_raw, "zlib_payload_bytes": total_packed, + "weighted_ratio_pct": 100.0 * total_packed / total_raw, + "raw_bytes_per_nonempty_tile_median": statistics.median(all_raw), + "decode_ms_per_tile_median": statistics.median(all_decode), + "decode_ms_per_tile_p95": percentile(all_decode, 0.95), + "lod": lod_result, + } + result["process_peak_rss_kib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + out = Path("Saved/Linux/research/R7_python_zlib_probe.json") + out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + for name, info in result["files"].items(): + print(f"{name}: tiles={info['nonempty_tile_records']}/{info['tile_records']} " + f"raw={info['raw_payload_bytes']} packed={info['zlib_payload_bytes']} " + f"ratio={info['weighted_ratio_pct']:.1f}% " + f"decode_p50/p95={info['decode_ms_per_tile_median']:.3f}/" + f"{info['decode_ms_per_tile_p95']:.3f}ms") + print(" LOD raw B/tile median | zlib B/tile median | decode p50/p95 ms") + for lod, row in info["lod"].items(): + print(f" {lod}: {row['raw_bytes_median']:.0f} | {row['packed_bytes_median']:.0f} | " + f"{row['decode_ms_median']:.3f}/{row['decode_ms_p95']:.3f}") + print(f"peak_rss={result['process_peak_rss_kib']} KiB; JSON={out}") + +if __name__ == "__main__": + main() diff --git a/Standalone/Research/WorldAssemblerM3/Research7/shared-notes.md b/Standalone/Research/WorldAssemblerM3/Research7/shared-notes.md new file mode 100644 index 0000000..031f12d --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Research7/shared-notes.md @@ -0,0 +1,2 @@ +# Shared notes (researcher R7 <-> implementer L40), newest at bottom +