diff --git a/Standalone/Core/src/ShapeWorldInternal.h b/Standalone/Core/src/ShapeWorldInternal.h index 5b11c50..8626d7e 100644 --- a/Standalone/Core/src/ShapeWorldInternal.h +++ b/Standalone/Core/src/ShapeWorldInternal.h @@ -3563,12 +3563,20 @@ public: double LocalPrimitiveSeconds = 0.0; double LocalLineBinSeconds = 0.0; double CandidateMergeSeconds = 0.0; + double PackedSignBuildSeconds = 0.0; + double PassageSweepPrepSeconds = 0.0; + double CandidateSweepPrepSeconds = 0.0; + double VisualDetailPrepSeconds = 0.0; double DetailApplySeconds = 0.0; double SignEdgeDistanceSeconds = 0.0; double PassageValidationSeconds = 0.0; double SharedFieldSeconds = 0.0; + double OwnerShapeWorkerSeconds = 0.0; double FarPrimitiveSeconds = 0.0; + double FarParentShapeSeconds = 0.0; + double FarSharedPrimitiveSeconds = 0.0; double FarBinSeconds = 0.0; + FParallelTiming FarParentGeometryLines; FParallelTiming RockIntervalLines; FParallelTiming CandidateAxisLines; FParallelTiming CandidateMergeLines; @@ -3701,7 +3709,6 @@ public: size_t capacity() const {return bHeap?Storage.Heap.Capacity:InlineCapacity;} const uint32* begin() const {return bHeap?Storage.Heap.Data:Storage.Inline.data();} const uint32* end() const {return begin()+Count;} - private: void Release() { @@ -3734,6 +3741,7 @@ public: static constexpr float StrataCaveBlendRadius = 12.0f; static constexpr int32 LineBinSize = 32; static constexpr int32 SpatialBinSize = 256; + static constexpr int32 FarSpatialBinSize = 512; bool Build(FStandaloneOwnerWorld& World, const FIntVector& MinVoxels, const FIntVector& MaxVoxels, const FIntVector& ClipMinVoxels, @@ -3779,7 +3787,11 @@ public: auto Stage=Clock::now(); BuildLayerBounds(); auto Substage=Clock::now(); - BuildOwnerPrimitives(); + const auto& OwnerLayout=Owner->Manager->GetLayout(); + if(WorkerCount>1&&Scheduler&&OwnerLayout.Num()>1) + BuildOwnerPrimitivesParallel(); + else + BuildOwnerPrimitives(); Stats.LocalPrimitiveSeconds=std::chrono::duration(Clock::now()-Substage).count(); Substage=Clock::now(); if(RoundTunnelsEnabled) @@ -3798,15 +3810,23 @@ public: WordsPerZ=(NZ+63u)/64u; RockWords.assign(static_cast(Size[0])*Size[1]*WordsPerZ,0); SurfaceDistanceQ=std::make_unique_for_overwrite(static_cast(NodeCount)); + Substage=Clock::now(); BuildPackedRockIntervals(Stats.RockIntervalLines); + Stats.PackedSignBuildSeconds=std::chrono::duration(Clock::now()-Substage).count(); // Native density queries for non-heightfield strates use the exact interval // evaluator. Build those native intervals before scanning passage support; // the scan itself still completes before any resolved column is composed. + Substage=Clock::now(); BuildPassageVoidCrossingPlan(); + Stats.PassageSweepPrepSeconds=std::chrono::duration(Clock::now()-Substage).count(); + Substage=Clock::now(); BuildCandidateBands(Stats.CandidateAxisLines,Stats.CandidateMergeLines, Stats.CandidateMergeSeconds); + Stats.CandidateSweepPrepSeconds=std::chrono::duration(Clock::now()-Substage).count(); + Substage=Clock::now(); if(!BuildGpuVisualDetailNodeSets(InGpuVisualDetailTiles,InCpuVisualDetailTiles,Error)) return false; + Stats.VisualDetailPrepSeconds=std::chrono::duration(Clock::now()-Substage).count(); Stats.IntervalSeconds=std::chrono::duration(Clock::now()-Stage).count(); Stage=Clock::now(); @@ -3861,6 +3881,14 @@ public: Stats.DetailApplySeconds*1000.0,Stats.SignEdgeDistanceSeconds*1000.0, Stats.PassageValidationSeconds*1000.0,Stats.SharedFieldSeconds*1000.0, Stats.FarShapeSeconds*1000.0); + std::printf("[ShapeBreakdown] owner_shapes=%.3fms owner_shape_worker=%.3fms owner_bins=%.3fms packed_signs=%.3fms passage_prep=%.3fms candidate_sweeps=%.3fms visual_detail_prep=%.3fms detail_band=%.3fms sign_edge_repair=%.3fms route_validation=%.3fms far_parent_shapes=%.3fms far_shared_primitives=%.3fms far_bins=%.3fms\n", + Stats.LocalPrimitiveSeconds*1000.0,Stats.OwnerShapeWorkerSeconds*1000.0, + Stats.LocalLineBinSeconds*1000.0, + Stats.PackedSignBuildSeconds*1000.0,Stats.PassageSweepPrepSeconds*1000.0, + Stats.CandidateSweepPrepSeconds*1000.0,Stats.VisualDetailPrepSeconds*1000.0, + Stats.DetailApplySeconds*1000.0,Stats.SignEdgeDistanceSeconds*1000.0, + Stats.PassageValidationSeconds*1000.0,Stats.FarParentShapeSeconds*1000.0, + Stats.FarSharedPrimitiveSeconds*1000.0,Stats.FarBinSeconds*1000.0); std::printf("[ShapeAllocation] words=%.2fMiB candidate_axis_words=%.2fMiB proxy=%.2fMiB primitives=%.2fMiB spatial_bins=%llu spatial_bin_refs=%llu spatial_bin_capacity=%llu inline_bins=%llu overflow_bins=%llu far_primitive_build=%.3fms far_bin_rebuild=%.3fms\n", static_cast(WordBytes)/1048576.0,static_cast(CandidateAxisBytes)/1048576.0, static_cast(Stats.SignNodes*sizeof(int16))/1048576.0, @@ -4468,16 +4496,26 @@ public: if(FarBuilt)return; auto Stage=std::chrono::steady_clock::now(); BuildFarParentShapes(); - RebuildSharedFieldPrimitives(); - Stats.FarPrimitiveSeconds=std::chrono::duration(std::chrono::steady_clock::now()-Stage).count(); + Stats.FarParentShapeSeconds=std::chrono::duration( + std::chrono::steady_clock::now()-Stage).count(); Stage=std::chrono::steady_clock::now(); - BuildSpatialBins(SpatialBinSize,false); - Stats.FarBinSeconds=std::chrono::duration(std::chrono::steady_clock::now()-Stage).count(); + RebuildSharedFieldPrimitives(); + Stats.FarSharedPrimitiveSeconds=std::chrono::duration( + std::chrono::steady_clock::now()-Stage).count(); + Stats.FarPrimitiveSeconds=Stats.FarParentShapeSeconds+Stats.FarSharedPrimitiveSeconds; + Stage=std::chrono::steady_clock::now(); + BuildSpatialBins(FarSpatialBinSize,false); + Stats.FarBinSeconds=std::chrono::duration( + std::chrono::steady_clock::now()-Stage).count(); Stats.FarShapeSeconds=Stats.FarPrimitiveSeconds+Stats.FarBinSeconds; Stats.PrimitiveCount=Primitives.size();FarBuilt=true; std::printf("[FarShapeStage] parent_shapes=%.3f ms spatial_bins=%.3f ms total=%.3f ms primitives=%llu\n", Stats.FarPrimitiveSeconds*1000.0,Stats.FarBinSeconds*1000.0,Stats.FarShapeSeconds*1000.0, static_cast(Stats.PrimitiveCount)); + std::printf("[FarShapeBreakdown] parent_generation=%.3fms parent_geometry_wall=%.3fms parent_geometry_worker=%.3fms shared_fields=%.3fms bin_rebuild=%.3fms\n", + Stats.FarParentShapeSeconds*1000.0,Stats.FarParentGeometryLines.WallSeconds*1000.0, + Stats.FarParentGeometryLines.WorkerSeconds*1000.0, + Stats.FarSharedPrimitiveSeconds*1000.0,Stats.FarBinSeconds*1000.0); std::fflush(stdout); } @@ -11277,14 +11315,112 @@ private: -FloorAt(P,X-UX*Epsilon,Y-UY*Epsilon))/(2.0*Epsilon); } - void BuildOwnerPrimitives() + void AccumulateOwnerPrimitiveStats(const FBuildStats& Other) + { + Stats.RoomCount+=Other.RoomCount; + Stats.RoomFloorJoinCount+=Other.RoomFloorJoinCount; + Stats.TunnelCount+=Other.TunnelCount; + if(Other.MaxTunnelBendDegrees>Stats.MaxTunnelBendDegrees) + { + Stats.MaxTunnelBendDegrees=Other.MaxTunnelBendDegrees; + Stats.MaxTunnelBendVoxels=Other.MaxTunnelBendVoxels; + Stats.MaxTunnelBendDirection=Other.MaxTunnelBendDirection; + } + Stats.LegacySegmentRadiusStep=std::max(Stats.LegacySegmentRadiusStep, + Other.LegacySegmentRadiusStep); + Stats.MaxCapsuleJoinFloorDelta=std::max(Stats.MaxCapsuleJoinFloorDelta, + Other.MaxCapsuleJoinFloorDelta); + Stats.MaxMarkedFloorSlopeDelta=std::max(Stats.MaxMarkedFloorSlopeDelta, + Other.MaxMarkedFloorSlopeDelta); + Stats.MaxRawUnmarkedFloorSlopeDelta=std::max(Stats.MaxRawUnmarkedFloorSlopeDelta, + Other.MaxRawUnmarkedFloorSlopeDelta); + Stats.SmoothedProfileRadiusStep=std::max(Stats.SmoothedProfileRadiusStep, + Other.SmoothedProfileRadiusStep); + Stats.MaxFloorCurveDeviation=std::max(Stats.MaxFloorCurveDeviation, + Other.MaxFloorCurveDeviation); + Stats.MaxFloorProfileEndpointValueError=std::max( + Stats.MaxFloorProfileEndpointValueError,Other.MaxFloorProfileEndpointValueError); + Stats.MaxFloorProfileEndpointSlopeError=std::max( + Stats.MaxFloorProfileEndpointSlopeError,Other.MaxFloorProfileEndpointSlopeError); + Stats.MaxCapsuleJoinRadiusDelta=std::max(Stats.MaxCapsuleJoinRadiusDelta, + Other.MaxCapsuleJoinRadiusDelta); + Stats.MaxCapsuleJoinRadiusSlopeDelta=std::max(Stats.MaxCapsuleJoinRadiusSlopeDelta, + Other.MaxCapsuleJoinRadiusSlopeDelta); + Stats.MaxCapsuleJoinFloorSlopeDelta=std::max(Stats.MaxCapsuleJoinFloorSlopeDelta, + Other.MaxCapsuleJoinFloorSlopeDelta); + Stats.MaxUnmarkedFloorSlopeDelta=std::max(Stats.MaxUnmarkedFloorSlopeDelta, + Other.MaxUnmarkedFloorSlopeDelta); + Stats.MaxUnmarkedFloorDirectionSlopeDelta=std::max( + Stats.MaxUnmarkedFloorDirectionSlopeDelta,Other.MaxUnmarkedFloorDirectionSlopeDelta); + Stats.MarkedFloorSlopeJoins+=Other.MarkedFloorSlopeJoins; + Stats.CurvedFloorSegments+=Other.CurvedFloorSegments; + Stats.CapsuleJoinCount+=Other.CapsuleJoinCount; + } + + void BuildOwnerPrimitivesParallel() + { + const auto& Layout=Owner->Manager->GetLayout(); + struct FStratumResult + { + std::vector Primitives; + FBuildStats Stats; + }; + struct FDispatch + { + FStandaloneShapeWorld* Parent=nullptr; + const TArray* Layout=nullptr; + std::vector* Results=nullptr; + }; + const size_t StratumCount=static_cast(Layout.Num()); + std::vector Results(StratumCount); + FDispatch Dispatch{this,&Layout,&Results}; + const auto BuildStratum=[](void* Opaque,size_t Index,size_t) + { + auto& Context=*static_cast(Opaque); + const FStrateSlot& Slot=(*Context.Layout)[Index]; + if(!Slot.Definition)return; + FStandaloneShapeWorld Partial; + Partial.Owner=Context.Parent->Owner; + Partial.WorldAssembly=Context.Parent->WorldAssembly; + Partial.ResolvedLayout=Context.Parent->ResolvedLayout; + Partial.LayerRanges=Context.Parent->LayerRanges; + Partial.LayerFields=Context.Parent->LayerFields; + for(int32 Axis=0;Axis<3;++Axis) + { + Partial.Min[Axis]=Context.Parent->Min[Axis]; + Partial.Max[Axis]=Context.Parent->Max[Axis]; + } + Partial.BuildOwnerPrimitives(Slot.StrateIndex,true,false); + FStratumResult& Result=(*Context.Results)[Index]; + Result.Primitives=std::move(Partial.Primitives); + Result.Stats=Partial.Stats; + }; + std::array WorkerSeconds{}; + const size_t WorkerLimit=std::min(static_cast(WorkerCount),StratumCount); + Stats.OwnerShapeWorkerSeconds=Scheduler->parallel_for(StratumCount,1,WorkerLimit, + &Dispatch,BuildStratum,WorkerSeconds.data(),WorkerSeconds.size()); + size_t PrimitiveCount=0; + for(const FStratumResult& Result:Results)PrimitiveCount+=Result.Primitives.size(); + Primitives.reserve(Primitives.size()+PrimitiveCount); + for(FStratumResult& Result:Results) + { + AccumulateOwnerPrimitiveStats(Result.Stats); + Primitives.insert(Primitives.end(), + std::make_move_iterator(Result.Primitives.begin()), + std::make_move_iterator(Result.Primitives.end())); + } + BuildOwnerPrimitives(-1,false,true); + } + + void BuildOwnerPrimitives(int32 OnlyStrateIndex=-1,bool bBuildStrata=true, + bool bBuildPassages=true) { const auto& Layout=Owner->Manager->GetLayout(); const float MinX=static_cast(Min[0]-256),MinY=static_cast(Min[1]-256); const float MaxX=static_cast(Max[0]+256),MaxY=static_cast(Max[1]+256); - for(const FStrateSlot& Slot:Layout) + if(bBuildStrata)for(const FStrateSlot& Slot:Layout) { - if(!Slot.Definition)continue; + if(!Slot.Definition||(OnlyStrateIndex>=0&&Slot.StrateIndex!=OnlyStrateIndex))continue; const FIntVector MidChunk(0,0,(Slot.TopChunkZ+Slot.BottomChunkZ)/2); const FStrateGenerationParams Params=Owner->Manager->GetGenerationParams(MidChunk); ActiveBlendRadius=0.0; @@ -11506,6 +11642,7 @@ private: // Reproduire dans chaque strate la salle d'origine finie et ancrée en haut. AddOriginLandingRoom(FieldForStrate(Slot.StrateIndex)); } + if(!bBuildPassages)return; const double PassageJoinBlend=ActiveBlendRadius;ActiveBlendRadius=JoinBlendRadius; ActiveFloorBlendRadius=FloorClipBlendRadius; for(const FVoxelPassage& P:Owner->Manager->GetPassages()) @@ -11786,7 +11923,7 @@ private: ActiveSpatialBinSize=BinResolution; // Primitive bounds overlap multiple cells. Reserve from the measured bins-per-primitive // density for each grid instead of repeatedly rehashing from a 2x primitive guess. - const size_t BinsPerPrimitive=bBuildAxisBins?96u:24u; + const size_t BinsPerPrimitive=bBuildAxisBins?96u:(BinResolution>=FarSpatialBinSize?8u:24u); SpatialBins.reserve(std::max(Primitives.size()*BinsPerPrimitive,1u)); if(!bBuildAxisBins) { @@ -11812,7 +11949,8 @@ private: B.Items[static_cast(IB-B.MinB)*B.CountA+(IA-B.MinA)].push_back(Id); } } - for(uint32 Id=0;Id(BinResolution); const double AxisStep=std::max(1.0,BinWidth*0.5); @@ -11874,17 +12012,28 @@ private: PrimitiveBins.end()); for(const FBinKey& Key:PrimitiveBins) { - SpatialBins[Key].push_back(Id); - if(bCriticalAir)CriticalAirBins[Key].push_back(Id); + AddSpatial(Key,Id); + if(bCriticalAir)AddCritical(Key,Id); } - continue; + return; } for(int32 Z=Z0;Z<=Z1;++Z)for(int32 Y=Y0;Y<=Y1;++Y)for(int32 X=X0;X<=X1;++X) { const FBinKey Key{X,Y,Z}; - SpatialBins[Key].push_back(Id); - if(bCriticalAir)CriticalAirBins[Key].push_back(Id); + AddSpatial(Key,Id); + if(bCriticalAir)AddCritical(Key,Id); } + }; + + const auto AddSpatial=[&](const FBinKey& Key,uint32 Id) + {SpatialBins[Key].push_back(Id);}; + const auto AddCritical=[&](const FBinKey& Key,uint32 Id) + {CriticalAirBins[Key].push_back(Id);}; + const auto AddUnbinned=[&](uint32 Id) + {UnbinnedSpatialPrimitiveIds.push_back(Id);}; + for(uint32 Id=0;Id Axis0={1,0,0},Axis1={0,1,0},Axis2={0,0,1}; - AddEllipsoid(P.Center,{P.Rx,P.Ry,P.Rz},Axis0,Axis1,Axis2,true); } } - for(const FParent& P:Parents) + struct FParentGeometry { + FPrimitive Ellipsoid; + std::vector Connectors; + std::vector Segments; + }; + std::vector ParentGeometry(Parents.size()); + ParallelLines(Parents.size(),[&](size_t ParentIndex,int32) + { + const FParent& P=Parents[ParentIndex]; + FParentGeometry& Geometry=ParentGeometry[ParentIndex]; + FStandaloneShapeWorld Builder; + Builder.ActiveBlendRadius=ActiveBlendRadius; + Builder.AddEllipsoid(P.Center,{P.Rx,P.Ry,P.Rz},{1,0,0},{0,1,0},{0,0,1},true); + Geometry.Ellipsoid=std::move(Builder.Primitives.back()); + Builder.Primitives.clear(); + Geometry.Connectors.reserve(2); + Geometry.Segments.reserve(2); for(const auto [DX,DY]:std::array,2>{{{1,0},{0,1}}}) { const auto It=ByCell.find(Key(P.GX+DX,P.GY+DY,P.Strate)); @@ -11948,11 +12111,26 @@ private: const int32 MinY=static_cast(std::floor(std::min(A.Y,B.Y)-64.0)); const int32 MaxY=static_cast(std::ceil(std::max(A.Y,B.Y)+64.0)); if(MaxX>=Min[0]&&MinX<=Max[0]&&MaxY>=Min[1]&&MinY<=Max[1])continue; - AddTaperedCapsule(A,B,64.0f,64.0f,true); - Primitives.back().BlendRadius=0.0; - FarConnectorSegments.push_back({A,B,64.0f}); + Builder.AddTaperedCapsule(A,B,64.0f,64.0f,true); + Builder.Primitives.back().BlendRadius=0.0; + Geometry.Connectors.push_back(std::move(Builder.Primitives.back())); + Builder.Primitives.clear(); + Geometry.Segments.push_back({A,B,64.0f}); } - } + },&Stats.FarParentGeometryLines); + size_t ConnectorCount=0; + for(const FParentGeometry& Geometry:ParentGeometry) + ConnectorCount+=Geometry.Connectors.size(); + Primitives.reserve(Primitives.size()+Parents.size()+ConnectorCount); + FarConnectorSegments.reserve(FarConnectorSegments.size()+ConnectorCount); + for(FParentGeometry& Geometry:ParentGeometry) + Primitives.push_back(std::move(Geometry.Ellipsoid)); + for(FParentGeometry& Geometry:ParentGeometry) + for(FPrimitive& Connector:Geometry.Connectors) + Primitives.push_back(std::move(Connector)); + for(FParentGeometry& Geometry:ParentGeometry) + FarConnectorSegments.insert(FarConnectorSegments.end(), + Geometry.Segments.begin(),Geometry.Segments.end()); BuildFarConnectorSpatialBins(); if(!Parents.empty()) std::printf("[FarParents] pitch=%d rooms=%zu links=%llu clip=(%d,%d)..(%d,%d)\n", diff --git a/Standalone/Research/WorldAssemblerM3/Linux105/AnalyzeL105Timeline.py b/Standalone/Research/WorldAssemblerM3/Linux105/AnalyzeL105Timeline.py new file mode 100644 index 0000000..186f7d4 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux105/AnalyzeL105Timeline.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Analyze L105 worker-idle phases and nested shape-build timers.""" +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l105/timeline" +EVENT = re.compile( + r"^\[CpuTileTimeline\] event=(\S+) key=\(([^,]+),([^,]+),([^,]+),lod(-?\d+)\) " + r"at_ms=([0-9.]+).*?worker_id=(-?\d+) queue_depth=(\d+) active_workers=(\d+) " + r"priority_rank=(\d+)", re.M) +COLD = re.compile(r"^\[ColdTimeline\] event=(\S+) edge=(\S+) at_ms=([0-9.]+)", re.M) +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") + + +def run(path: Path) -> dict: + content = path.read_text(encoding="utf-8", errors="replace") + events = [] + for match in EVENT.finditer(content): + event, x, y, z, lod, ms, worker, queue, active, rank = match.groups() + events.append({"event": event, "lod": int(lod), "at_ms": float(ms), + "worker": int(worker), "queue": int(queue), + "active": int(active), "rank": int(rank)}) + cold = {f"{event}_{edge}": float(ms) for event, edge, ms in COLD.findall(content)} + idle_start: dict[int, float] = {} + idle_intervals = [] + for event in sorted((item for item in events if item["event"] in ("idle_begin", "idle_end")), + key=lambda item: item["at_ms"]): + if event["event"] == "idle_begin": + idle_start[event["worker"]] = event["at_ms"] + elif event["worker"] in idle_start: + idle_intervals.append((idle_start.pop(event["worker"]), event["at_ms"])) + near = NEAR.search(content) + pipeline = PIPELINE.search(content) + config = re.search(r"^\[HeadlessConfig\] (.*)$", content, re.M) + if not events or near is None or pipeline is None or config is None: + raise RuntimeError(f"incomplete timeline markers: {path}") + config_fields = dict(FIELDS.findall(config.group(1))) + workers = int(config_fields["workers"]) + whole_fields = dict(FIELDS.findall(pipeline.group(1))) + cold_events = {f"{event}_{edge}": float(ms) + for event, edge, ms in COLD.findall(content)} + phases = ( + ("full_pipeline", "desired_tile_set_requested", "lod0_2_done_pipeline_drain"), + ("base_shape_compile", "base_compile_begin", "base_compile_end"), + ("lod0_1_tile_drain", "lod0_1_admission_end", "lod0_1_done_publish_drained"), + ("far_parent_build", "far_parent_build_begin", "far_parent_build_end"), + ("lod2_queue_and_drain", "lod2_queue_begin", "lod2_done_pipeline_drain"), + ) + phase_rows = [] + for label, begin, end in phases: + start, finish = cold_events[begin], cold_events[end] + idle_ms = sum(max(0.0, min(b, finish) - max(a, start)) + for a, b in idle_intervals) + capacity_ms = max(0.0, finish - start) * workers + phase_rows.append({"phase": label, "wall_ms": finish - start, + "idle_worker_ms": idle_ms, + "idle_pct": 100.0 * idle_ms / capacity_ms if capacity_ms else 0.0, + "busy_pct": 100.0 * (1.0 - idle_ms / capacity_ms) + if capacity_ms else 0.0}) + + def timer(prefix: str) -> dict: + match = re.search(rf"^\[{prefix}\] (.*)$", content, re.M) + return dict(FIELDS.findall(match.group(1))) if match else {} + + requested = [item for item in events if item["event"] == "requested"] + admitted = [item for item in events if item["event"] == "admitted"] + lod01_started = [item for item in events if item["event"] == "started" and item["lod"] <= 1] + lod01_finished = [item for item in events if item["event"] == "finished" and item["lod"] <= 1] + return { + "log": str(path.relative_to(ROOT)), "workers": workers, + "source_stamp": re.search(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", + content, re.M).group(1), + "requested_lod01": sum(item["lod"] <= 1 for item in requested), + "admitted_lod01": sum(item["lod"] <= 1 for item in admitted), + "started_lod01": len(lod01_started), "finished_lod01": len(lod01_finished), + "near_tiles": int(near.group(1)), "whole_tiles": whole_fields["whole_tiles"], + "phase_timers": timer("ShapeBreakdown"), + "far_phase_timers": timer("FarShapeBreakdown"), + "worker_idle_phases": phase_rows, + } + + +def main() -> int: + logs = sorted(OUT.glob("*-w*-direct-measured-r1.log")) + if len(logs) != 4: + raise SystemExit(f"expected FI/TN W8/W12 traces, found {len(logs)}") + rows = [run(path) for path in logs] + (OUT / "analysis.json").write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") + with (OUT / "utilization.csv").open("w", newline="", encoding="utf-8") as stream: + fields = ["log", "workers", "phase", "wall_ms", "idle_worker_ms", "idle_pct", "busy_pct"] + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + for row in rows: + for phase in row["worker_idle_phases"]: + writer.writerow({"log": row["log"], "workers": row["workers"], **phase}) + print("[L105TimelineAnalysis] " + json.dumps(rows, sort_keys=True), flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux105/R45-simplification-audit.md b/Standalone/Research/WorldAssemblerM3/Linux105/R45-simplification-audit.md new file mode 100644 index 0000000..3a78372 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux105/R45-simplification-audit.md @@ -0,0 +1,83 @@ +# Ranked proposals + +The tunnel lesson has already been applied to the bore itself: the route is swept into a tile-local field. The strongest remaining opportunity is the floor/support work around that bore. Two lower-confidence targets are boundary-band evaluation and the full-shape startup barrier. + +The cost figures below are worker CPU allocations from the reports, not predicted wall-time savings. [MEASURED] The older Astra review included a “75%” passage-cost claim that it later identified as wrong; I use the newer census instead. [Astra1:3]() + +## 1. Put tunnel floors and mouth support into the bounded passage field + +**Today.** [CODE] The passage sweep already visits a tile-local grid, bounds work to a 4 m influence region, and min-combines tube distances. [ShapeWorldInternal.h:14395]() Support remains a separate per-sample path: it projects onto route segments, checks capsule clearance, can defer support until later bores are considered, and restores native ground near mouths. [ShapeWorldInternal.h:2619]() [ShapeWorldInternal.h:2936]() [ShapeWorldInternal.h:2998]() Mouth shaping is partly in the sweep already; native-ground restoration still has its own path. [ShapeWorldInternal.h:3006]() + +**Cost.** [MEASURED] Support accounts for an estimated 1.864 worker-seconds in FI and 1.623 in TN near LOD0–2. [Linux103:14]() L101 counted 10.6M / 8.37M support visits and 4.09M / 3.38M capsule checks for FI / TN. [Linux101:28]() [Linux101:29]() + +**Simpler construction.** [INFERENCE] Prototype a local floor ribbon or lofted cross-section as part of the per-tile passage field. Let route data describe the floor profile and mouth transition; write that bounded contribution alongside the tube. Keep the exact off-grid fallback: L100 showed that a missing lattice sample cannot prove a mesh query is outside the tunnel. [Linux100:9]() + +**World and risk.** [INFERENCE] Reusing the existing support formulas and order could preserve the same density field. A simpler cross-section will likely move floor geometry, especially at helical overlaps, short spans, and mouths. It must preserve native-ground support, headroom, and the checks that keep neighboring coils from becoming shelves. [ShapeWorldInternal.h:2944]() + +**Effort: medium-high. Falsifier:** Prototype one ordinary passage tile and one overlapping-coil tile. Compare lattice density, mesh crossings, and capsule clearance to the current field. Reject the approach if support depends on pointwise native-solid or neighboring-bore checks that the local profile cannot encode, or if any crossing or walkability result changes. + +[PRIOR ART] OpenVDB’s tube tools construct narrow-band tube and capsule SDFs, which supports the local-write pattern; it does not establish that VoxelForge’s floor and mouth semantics can be replaced unchanged. [OpenVDB LevelSetTubes](https://www.openvdb.org/documentation/doxygen/LevelSetTubes_8h.html) [OpenVDB narrow-band documentation](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/doc/doc.txt) + +## 2. Compute boundary-band geometry once per XY column + +**Today.** [CODE] Boundary-band composition already has tile and Z-range culls. Inside the active window, it evaluates displaced transition bands from X/Y and applies the resulting vertical signed distance at the queried Z. [ShapeWorldInternal.h:1485]() [ShapeWorldInternal.h:1549]() [ShapeWorldInternal.h:1612]() + +**Cost.** [MEASURED] The census allocates about 0.773 worker-seconds to bands for FI and 0.781 for TN. [Linux103:16]() + +**Simpler construction.** [INFERENCE] For each tile column, evaluate the transition bands and passage fade once, then reuse those bounds for the affected Z samples. This turns repeated XY-dependent work into a column-local operand without changing the band’s Z formula. + +**World and risk.** [INFERENCE] This can be same-world if it preserves the current fixed-point rounding, native-rock preservation rule, FI air-gap exception, and composition order. The main risk is a subtly different band at an interface or passage mouth. + +**Effort: low-medium. Falsifier:** Compare exact lattice densities for hard and displaced transitions, an FI air gap, and a passage-faded mouth. Reject on any density mismatch or lost surface crossing. + +## 3. Let certified SurfaceWorld tiles start before the full shape is ready + +**Today.** [CODE] `World::compile` publishes the shape as ready only after the complete shape build succeeds; tile-region compilation also requires a built base shape. [VoxelForgeCore.cpp:1442]() [VoxelForgeCore.cpp:1582]() The L10 SurfaceWorld fixture’s first tile uses the native heightfield before packed-shape lookup, but the API still waits for the full build. [Linux10:39]() + +**Cost.** [MEASURED] That fixture spent 222.59 ms wall time and 856.79 ms process CPU compiling the shape; its first tile began at 299.62 ms. The report gives 222.6 ms as the maximum first-tile delay this could remove for that case. [Linux10:18]() [Linux10:21]() [Linux10:58]() + +**Simpler construction.** [INFERENCE] Publish immutable shape pages by capability, and allow only tile requests whose dependency set proves they cannot read unfinished shape data. Keep the full readiness gate for other paths. + +**World and risk.** [INFERENCE] Correct dependency classification would leave generated density unchanged. Partial publication could expose unwritten signs, proxies, bins, or far fields; the proof must cover fallback and seam queries too. + +**Effort: high. Falsifier:** Trace every field read from a candidate SurfaceWorld tile. Reject if it can touch unfinished data, or if the measured first-tile start does not move earlier enough to justify the readiness state. + +## 4. Reduce repeated far-shell crossing refinement only where the boundary field defines the surface + +**Today.** [CODE] Missing far-shell faces are patched with local geometry, but the patch validates candidate positions against the source field. `FindFarStratumCrossings` samples at 1 m intervals and binary-refines crossings; nearest-crossing refinement also makes repeated density queries. [FarShellsInternal.h:1069]() [FarShellsInternal.h:1098]() [FarShellsInternal.h:1270]() + +**Cost.** [MEASURED] L103 counted about 1.13M / 1.15M far-shell stratum-refinement queries per FI / TN whole-view run. [Linux103:57]() This is a query count, not an isolated time. The separate hit-search cost was already reduced by L104’s 3D bins, from estimated 1.73 / 1.85 to 0.37 / 0.34 worker-seconds. [Linux104:20]() + +**Simpler construction.** [INFERENCE] Where a patch crosses a resolved boundary band, construct its local surface directly from the band’s XY boundary formula and reserve exact source-field refinement for ambiguous columns. Do not skip the patch: the report says near tiles still need boundary patches. [Linux104:7]() + +**World and risk.** [INFERENCE] The patch may change or omit a surface where the actual crossing is controlled by host-rock density, passage fade, or another field rather than the boundary formula. Effort is high because the optimization must preserve coverage. + +**Falsifier:** Compare all patched crossing positions and coverage against the current source-field refinement. Reject if any face disappears, moves beyond the existing tolerance, or if query time does not materially fall. + +## 5. Compact sign-distance proxies only if active memory use is actually large + +**Today.** [CODE] Shape compilation reserves one `int16` proxy slot per shape node, while candidate bits decide which positions have usable proxy values. [ShapeWorldInternal.h:3810]() [ShapeWorldInternal.h:12889]() + +**Cost.** [MEASURED] L10 reports 263.57 MiB of proxy address capacity and 610.3 MiB peak RSS for the fixture. The report does not establish that the whole proxy allocation is resident or how many entries are active. [Linux10:37]() [Linux10:5]() + +**Simpler construction.** [INFERENCE] If most slots are inactive and resident, store active proxy entries compactly behind the candidate mask. This is a memory proposal, not a demonstrated generation-speed win. + +**Effort: medium. Falsifier:** First measure active entries and resident pages. Stop if the array is mostly nonresident or compact lookup costs more than it saves in memory pressure or readiness. + +# What I would leave alone for now + +- **Columns.** [MEASURED] L101 counted 11,804 column-distance calls for FI and zero for TN, far below millions of support visits. [Linux101:28]() [Linux101:29]() They are already constrained to prepared passage crossings and Z overlap. [CODE] Their separate cost does not justify folding them into the support change yet. [ShapeWorldInternal.h:2000]() + +- **Cavern graphs and caps.** [CODE] Legacy and native cavern evaluation are selected by stratum; `ApplyResolvedCavernGraph` dispatches to the native evaluator when the stratum advertises that field. [ShapeWorldInternal.h:10182]() [ShapeWorldInternal.h:10339]() Native caves combine floor support with the void; the final floor-cap path is retained for legacy. [ShapeWorldInternal.h:10512]() [MEASURED] Graph, caps, and edits together are only 0.084 s FI / 0.053 s TN in the current census. [Linux103:18]() Consolidating these means a migration between generation contracts, not removing duplicate work from each query. + +- **Island roots and connectors.** [CODE] Island and connector bounds already produce tile-local feature lists. [FloatingIslandsInternal.h:2318]() The implementation explicitly chooses a full scan for the measured four-island plan and candidate grouping for larger plans. [FloatingIslandsInternal.h:2323]() Replacing that structure with one primitive risks changing the island silhouette without an identified hot cost. + +- **Sign-edge repair.** [CODE] `CompleteSignEdgeDistances` supplies exact distances at missing sign edges and corrects packed signs when they disagree with the analytic field. [ShapeWorldInternal.h:12872]() [ShapeWorldInternal.h:12900]() [MEASURED] L94’s parallel repair preserved hashes and cut SW repair from 1,753.9 ms to 86.4 ms headless; FI had zero endpoint corrections. [Linux94:30]() [Linux94:32]() Removing the repair risks wrong mesh signs; it is already parallelized. + +- **Skirts.** [CODE] Production emits simple extruded boundary curtains. [VoxelMarchingCubesMesher.cpp:1499]() The standalone research mesher adds density clipping and recursive refinement, so its L103 skirt timing is not a production Unreal skirt cost. [VoxelMarchingCubesMesher.cpp:2543]() [MEASURED] That standalone stage is about 0.656 / 0.661 worker-seconds FI / TN. [Linux103:20]() [INFERENCE] If replacing curtains, pilot transition cells only on certified 2:1 LOD interfaces; Transvoxel is prior art for local 2:1 transitions, but does not cover every boundary the current skirts cover. [Transvoxel](https://transvoxel.org/) Falsify that pilot if it introduces cracks at a 2:1 interface or changes wall/floor collision. + +- **Route planners and schedulers.** [CODE] The adapter’s `BuildFiniteSurfaceRoute` searches supported surface routes over a heightfield; Core’s `PlanResolvedLayoutPassagesV1` plans inter-stratum passages. [VoxelForgeCoreAdapter.cpp:248]() [ResolvedLayoutInternal.h:2237]() They solve different route problems. Likewise, Unreal’s Core scheduler wraps intra-build `ParallelFor`, while streaming admission schedules tile tasks. [VoxelWorld.cpp:116]() [VoxelWorld.cpp:235]() [MEASURED] L97’s hard cap of seven tasks roughly doubled near-settle time; keep the scheduling layers separate unless a specific queue cost is measured. [Linux97:17]() [Linux97:21]() + +- **More caches.** [MEASURED] The exact result cache regressed FI readiness by 47–62% and estimated 699 MB of storage; the trilinear cache also regressed FI/TN readiness and increased route visits. [Linux86:22]() [Linux88:27]() [INFERENCE] Prefer constructing a bounded operand once in the tile’s generation pass over memoizing repeated global queries. + +Read-only review only: no edits, builds, tests, or benchmarks were run. \ No newline at end of file diff --git a/Standalone/Research/WorldAssemblerM3/Linux105/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux105/REPORT.md new file mode 100644 index 0000000..82ba460 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux105/REPORT.md @@ -0,0 +1,49 @@ +# Linux 105 — serial startup phases + +## Changes and phase timings + +Owner-shape compilation now runs by stratum on the core parallel scheduler. Results are merged in owner-layout order. Far-parent ellipsoid and connector generation also uses parallel workers, then merges in canonical order. The far-parent spatial index uses larger 512-unit bins; the exact primitive predicate and insertion order remain unchanged. LOD2 still waits for far parents. + +| Phase | L104 median | L105 median | Change | +|---|---:|---:|---:| +| Base-shape compile | 239.1 ms | 202.8 ms | −15.2% | +| Far-parent build | 128.5 ms | 33.4 ms | −74.0% | + +L105 base-shape substage timers across FI/TN at W8/W12: + +| Substage | Range | +|---|---:| +| Owner shapes (parallel wall; aggregate worker time) | 22.1–23.8 ms; 47.8–51.7 worker-ms | +| Owner bins / visual detail prep | 1.18–1.40 ms / below 0.001 ms | +| Packed intervals/signs | 23.7–25.9 ms | +| Passage prep | 10.1–11.7 ms | +| Candidate sweeps | 23.3–27.6 ms | +| Surface/detail band | 54.0–93.2 ms | +| Sign-edge repair | 9.6–10.7 ms | +| Route validation | 26.5–29.3 ms | +| Far-parent geometry (parallel wall; aggregate worker time) | 2.4–2.7 ms; 16.2–18.2 worker-ms | +| Far shared fields / bin rebuild | 4.0–4.3 ms / 16.7–18.3 ms | + +The timeline tracks streaming tile workers. In L104 they were about 99.99% idle during the 232–251 ms base compile, 99.4% busy during LOD0–1, and idle throughout the 110–144 ms far-parent phase. In L105 the tile pool remains 99.96–99.99% idle during base compile while the separate core parallel scheduler performs owner-shape work (22–24 ms wall from 48–52 worker-ms). LOD0–1 is 99.2–99.6% busy; far-parent takes 33–34 ms with the tile pool idle; LOD2 is 92.8–95.9% busy. Full event data and the utilization breakdown are in `Saved/Linux/l105/timeline/` (`analysis.json`, `utilization.csv`, and four logs). + +## L80 readiness pairs + +Sixteen warmups total, then three interleaved measured runs per build and cell. Each cell has complete near and whole-view tile sets; values are medians in seconds, L104 → L105. + +| View | W8 near | W8 whole | W12 near | W12 whole | +|---|---:|---:|---:|---:| +| SW | 0.840 → 0.744 (−11.4%) | 1.957 → 1.835 (−6.2%) | 0.839 → 0.746 (−11.2%) | 1.878 → 1.762 (−6.2%) | +| FI | 1.597 → 1.556 (−2.6%) | 2.594 → 2.516 (−3.0%) | 1.514 → 1.420 (−6.2%) | 2.438 → 2.338 (−4.1%) | +| TN | 1.516 → 1.455 (−4.0%) | 2.475 → 2.341 (−5.4%) | 1.384 → 1.301 (−6.0%) | 2.228 → 2.140 (−3.9%) | +| VS | 1.085 → 1.024 (−5.6%) | 2.016 → 1.920 (−4.8%) | 0.917 → 0.849 (−7.4%) | 1.774 → 1.703 (−4.0%) | + +## Exactness and verification + +- All 24 hash runs passed: SW/FI/TN/VS × Clang/GCC × W1/W8/W12. Every tile hash and whole-view hash matched L104 and the Clang W1 reference. +- Candidate source stamp: `c6b2740c67bc96252ba57eb303b40d97d3c3451fcb4f1daed7bae773351681ab`. +- Fast tier: **PASS**, 278.4 s. +- Linux Unreal build: **Succeeded** with `-NoUBA -MaxParallelActions=4`; it compiled the modified core shim. Build log: `Saved/Linux/l105/unreal-build.log`. +- Three FI and three SW Unreal arrival traces: **6/6 passed**, exit 0. FI settled 1,216 near tiles in 1.545–1.562 s from first admission; SW settled 1,430 in 1.285–1.317 s. Unreal ran with `taskset -c 0-11`, without `prlimit`. +- Parallel full-bin assembly variants were retired after regressions. The retained serial exact bin rebuild measured 16.7–18.3 ms with the 512-unit far bins. + +Hash, timing, readiness, and arrival logs are under `Saved/Linux/l105/`. No commit was made. diff --git a/Standalone/Research/WorldAssemblerM3/Linux105/RunL105Hashes.py b/Standalone/Research/WorldAssemblerM3/Linux105/RunL105Hashes.py new file mode 100644 index 0000000..82ac4a4 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux105/RunL105Hashes.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Compare L105 exact tile hashes with L104 across all strata, compilers and workers.""" +from __future__ import annotations + +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l105/hashes" +DATA = ROOT / "Standalone/data/owner_world.json" +OLD = ROOT / "Saved/Linux/l104/hashes" +BINARIES = { + "clang": ROOT / "Saved/Linux/l105/bin/VoxelForgeCoreHeadless-l105-clang", + "gcc": ROOT / "Saved/Linux/l105/bin/VoxelForgeCoreHeadless-l105-gcc", +} +CAMERAS = { + "sw": ((8.0, 16.0, -372.2626953125), 2850), + "fi": ((25.8369140625, 3.19921875, -737.0), 2249), + "tn": ((19.166015625, -14.2001953125, -753.3740234375), 2233), + "vs": ((-20.0625, -5.068359375, -900.353515625), 2329), +} +TILE = re.compile( + r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) " + r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) " + r"topology=(0x[0-9a-f]+)$", re.M) +WHOLE = re.compile( + r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) " + r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M) + + +def parse(path: Path) -> tuple[dict, tuple[str, ...], str]: + content = path.read_text(encoding="utf-8", errors="replace") + tiles = {(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))): + m.group(5, 6, 7, 8) for m in TILE.finditer(content)} + whole, stamp = WHOLE.search(content), STAMP.search(content) + if whole is None or stamp is None or len(tiles) != int(whole.group(1)): + raise RuntimeError(f"incomplete hash output: {path}") + return tiles, whole.groups(), stamp.group(1) + + +def run(stratum: str, compiler: str, workers: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + log = OUT / f"{stratum}-{compiler}-w{workers}.log" + camera, expected_tiles = CAMERAS[stratum] + camera_arg = ",".join(f"{v:.12f}" for v in camera) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARIES[compiler]), str(DATA), "--world-assembler-prototype", + "--stream-benchmark", "--stream-pipeline", f"--workers={workers}", + "--clip-radius=3", "--max-clip-level=8", "--max-level=8", + f"--camera={camera_arg}", "--stream-hash-tiles", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + process = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + if process.returncode: + raise RuntimeError(f"hash process exited {process.returncode}: {log}") + tiles, whole, stamp = parse(log) + if len(tiles) != expected_tiles: + raise RuntimeError(f"expected {expected_tiles} tiles, got {len(tiles)}: {log}") + old_log = OLD / f"{stratum}-{compiler}-w{workers}.log" + old_tiles, old_whole, _old_stamp = parse(old_log) + clang_reference, clang_whole, _ = parse(OLD / f"{stratum}-clang-w1.log") + + def difference(reference: dict, candidate: dict) -> tuple[int, int]: + keys = set(reference) | set(candidate) + mismatches = sum(reference.get(key) != candidate.get(key) for key in keys) + return mismatches, len(keys) + + old_mismatches, old_keys = difference(old_tiles, tiles) + cross_mismatches, cross_keys = difference(clang_reference, tiles) + row = { + "stratum": stratum, "compiler": compiler, "workers": workers, + "source_stamp": stamp, "tiles": len(tiles), + "l104_same_cell_tile_mismatches": old_mismatches, + "l104_same_cell_key_count": old_keys, + "l104_same_cell_whole_hash_match": whole == old_whole, + "clang_w1_tile_mismatches": cross_mismatches, + "clang_w1_key_count": cross_keys, + "clang_w1_whole_hash_match": whole == clang_whole, + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT)), + } + print("[L105HashRun] " + json.dumps(row, sort_keys=True), flush=True) + row["tile_hashes"] = {"|".join(map(str, key)): value for key, value in tiles.items()} + return row + + +def main() -> int: + for path in (*BINARIES.values(), DATA): + if not path.is_file(): + raise SystemExit(f"missing input: {path}") + rows = [] + for stratum in CAMERAS: + for compiler in BINARIES: + for workers in (1, 8, 12): + rows.append(run(stratum, compiler, workers)) + (OUT / "runs-partial.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + failures = [row for row in rows if row["l104_same_cell_tile_mismatches"] + or not row["l104_same_cell_whole_hash_match"] + or row["clang_w1_tile_mismatches"] + or not row["clang_w1_whole_hash_match"]] + summary = {"runs": [{key: value for key, value in row.items() if key != "tile_hashes"} + for row in rows], "failed_runs": len(failures)} + (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", + encoding="utf-8") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux105/RunL105ReadinessPairs.py b/Standalone/Research/WorldAssemblerM3/Linux105/RunL105ReadinessPairs.py new file mode 100644 index 0000000..5731f68 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux105/RunL105ReadinessPairs.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""L80-protocol L104/L105 near and whole-view readiness pairs.""" +from __future__ import annotations + +import json +import re +import statistics +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l105/readiness-paired" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARIES = { + "before": ROOT / "Saved/Linux/l104/bin/VoxelForgeCoreHeadless-l104-clang", + "after": ROOT / "Saved/Linux/l105/bin/VoxelForgeCoreHeadless-l105-clang", +} +STRATA = { + "sw": {"camera": (8.0, 16.0, -372.2626953125), "near": 1408, "whole": 2850}, + "fi": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249}, + "tn": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233}, + "vs": {"camera": (-20.0625, -5.068359375, -900.353515625), "near": 1344, "whole": 2329}, +} +NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") + + +def run(stratum: str, workers: int, build: str, phase: str, replicate: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + suffix = f"-r{replicate}" if phase == "measured" else "" + log = OUT / f"{stratum}-w{workers}-{build}-{phase}{suffix}.log" + camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARIES[build]), str(DATA), "--world-assembler-prototype", + "--stream-benchmark", "--stream-pipeline", f"--workers={workers}", + "--clip-radius=3", "--max-clip-level=8", "--max-level=8", + f"--camera={camera}", "--profile-lod-work", "--stream-scheduler-policy=3", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + process = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + content = log.read_text(encoding="utf-8", errors="replace") + near, pipeline, config, stamp = (NEAR.search(content), PIPELINE.search(content), + CONFIG.search(content), STAMP.search(content)) + if process.returncode or not all((near, pipeline, config, stamp)): + raise RuntimeError(f"readiness run failed or missing marker: {log}; exit={process.returncode}") + expected = STRATA[stratum] + config_fields = dict(FIELDS.findall(config.group(1))) + pipeline_fields = dict(FIELDS.findall(pipeline.group(1))) + if int(config_fields["workers"]) != workers: + raise RuntimeError(f"worker mismatch: {log}") + if (int(near.group(1)), int(near.group(2))) != (expected["near"], expected["near"]): + raise RuntimeError(f"incomplete LOD0-2 set: {log}: {near.group(1)}/{near.group(2)}") + whole_done, whole_total = map(int, pipeline_fields["whole_tiles"].split("/")) + if (whole_done, whole_total) != (expected["whole"], expected["whole"]): + raise RuntimeError(f"incomplete whole view: {log}: {whole_done}/{whole_total}") + row = { + "stratum": stratum, "workers": workers, "build": build, + "phase": phase, "replicate": replicate, "source_stamp": stamp.group(1), + "near_tiles": expected["near"], "whole_tiles": whole_done, + "lod02_ready_s": float(near.group(3)), + "whole_view_ready_s": float(pipeline_fields["whole_view_ready"].removesuffix("s")), + "shape_s": float(pipeline_fields["shape"].removesuffix("s")), + "far_shape_s": float(pipeline_fields["far_shape"].removesuffix("s")), + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT)), + } + print("[L105ReadinessRun] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + for path in (*BINARIES.values(), DATA): + if not path.is_file(): + raise SystemExit(f"missing input: {path}") + cells = [(stratum, workers) for stratum in ("vs", "tn", "fi", "sw") + for workers in (12, 8)] + for stratum, workers in cells: + for build in ("before", "after"): + run(stratum, workers, build, "warmup", 0) + orders = (("before", "after"), ("after", "before"), ("before", "after")) + rows = [] + for stratum, workers in cells: + for replicate, order in enumerate(orders, 1): + for build in order: + rows.append(run(stratum, workers, build, "measured", replicate)) + (OUT / "runs-partial.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + summary = [] + for stratum in STRATA: + for workers in (8, 12): + group = [row for row in rows if row["stratum"] == stratum + and row["workers"] == workers] + item = {"stratum": stratum, "workers": workers, "measured_runs_per_build": 3} + for build in ("before", "after"): + subset = [row for row in group if row["build"] == build] + for metric in ("lod02_ready_s", "whole_view_ready_s"): + values = [row[metric] for row in subset] + item[f"{build}_{metric}"] = values + item[f"{build}_{metric}_median"] = statistics.median(values) + for metric in ("lod02_ready_s", "whole_view_ready_s"): + item[f"{metric}_delta_pct"] = 100.0 * ( + item[f"after_{metric}_median"] / item[f"before_{metric}_median"] - 1.0) + summary.append(item) + (OUT / "summary.json").write_text( + json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n", + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux105/RunL105Timeline.py b/Standalone/Research/WorldAssemblerM3/Linux105/RunL105Timeline.py new file mode 100644 index 0000000..92e7b67 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux105/RunL105Timeline.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Capture L105 cold startup, phase and worker-idle timelines.""" +from __future__ import annotations + +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l105/timeline" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l105/bin/VoxelForgeCoreHeadless-l105-clang" +STRATA = { + "fi": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249}, + "tn": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233}, +} +NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M) +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M) +STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") + + +def run(stratum: str, workers: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + log = OUT / f"{stratum}-w{workers}-direct-measured-r1.log" + camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s", + str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark", + "--stream-pipeline", f"--workers={workers}", "--clip-radius=3", + "--max-clip-level=8", "--max-level=8", f"--camera={camera}", + "--profile-lod-work", "--stream-scheduler-policy=3", "--cold-worker-timeline", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + process = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + content = log.read_text(encoding="utf-8", errors="replace") + near, pipeline, config, stamp = (NEAR.search(content), PIPELINE.search(content), + CONFIG.search(content), STAMP.search(content)) + if process.returncode or not all((near, pipeline, config, stamp)): + raise RuntimeError(f"timeline run failed or missing marker: {log}; exit={process.returncode}") + if not re.search(r"^\[CpuTileTimeline\] event=", content, re.M): + raise RuntimeError(f"missing CPU tile timeline: {log}") + if not re.search(r"^\[ColdTimeline\] event=", content, re.M): + raise RuntimeError(f"missing cold timeline: {log}") + config_fields = dict(FIELDS.findall(config.group(1))) + pipeline_fields = dict(FIELDS.findall(pipeline.group(1))) + spec = STRATA[stratum] + whole = tuple(map(int, pipeline_fields["whole_tiles"].split("/"))) + if int(config_fields["workers"]) != workers: + raise RuntimeError(f"worker mismatch: {log}") + if (int(near.group(1)), int(near.group(2))) != (spec["near"], spec["near"]): + raise RuntimeError(f"incomplete near set: {log}: {near.group(1)}/{near.group(2)}") + if whole != (spec["whole"], spec["whole"]): + raise RuntimeError(f"incomplete whole set: {log}: {whole}") + row = { + "stratum": stratum, "workers": workers, "source_stamp": stamp.group(1), + "near_tiles": spec["near"], "whole_tiles": spec["whole"], + "near_ready_s": float(near.group(3)), + "whole_view_ready_s": float(pipeline_fields["whole_view_ready"].removesuffix("s")), + "shape_s": float(pipeline_fields["shape"].removesuffix("s")), + "far_shape_s": float(pipeline_fields["far_shape"].removesuffix("s")), + "wall_s": round(time.monotonic() - started, 3), + "log": str(log.relative_to(ROOT)), + } + print("[L105TimelineRun] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + if not BINARY.is_file() or not DATA.is_file(): + raise SystemExit(f"missing input: {BINARY} or {DATA}") + rows = [run(stratum, workers) for stratum in STRATA for workers in (8, 12)] + (OUT / "runs.json").write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux105/RunL105UnrealArrival.py b/Standalone/Research/WorldAssemblerM3/Linux105/RunL105UnrealArrival.py new file mode 100644 index 0000000..5a8bcb3 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux105/RunL105UnrealArrival.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 +"""Run one pinned null-RHI SW/FI arrival for Linux 105.""" +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime +import json +import re +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +HOST = Path("/home/jahni/VFHost") +HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l105" +HOST_SAVED = HOST / "Saved/Linux/l105" +EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd") +OUT = ROOT / "Saved/Linux/l105/unreal-arrival" +SPAWNS_M = { + "FI": (25.8369140625, 3.19921875, -737.0), + "SW": (25.8369140625, 3.19921875, -200.0), +} +MOVE_START_S = 20.0 +MOVE_DURATION_S = 12.0 +EXIT_S = MOVE_START_S + MOVE_DURATION_S + 5.0 +RUN_TIMEOUT_S = 1740 + + +def read_events(path: Path) -> list[dict]: + events = [] + if path.is_file(): + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith("{"): + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + pass + return events + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + return ordered[min(len(ordered) - 1, int(round((len(ordered) - 1) * fraction)))] + + +def sample_unreal_peak_rss() -> tuple[int, int] | None: + executable = str(EDITOR).encode() + project = str(HOST / "VFHost.uproject").encode() + peak: tuple[int, int] | None = None + for process in Path("/proc").glob("[0-9]*"): + try: + command_line = (process / "cmdline").read_bytes() + if executable not in command_line or project not in command_line: + continue + status = (process / "status").read_text(encoding="ascii", errors="ignore") + match = re.search(r"^VmHWM:\s+(\d+)\s+kB$", status, re.MULTILINE) + if match: + rss_kib = int(match.group(1)) + if peak is None or rss_kib > peak[0]: + peak = (rss_kib, int(process.name)) + except (FileNotFoundError, PermissionError, ProcessLookupError, OSError): + continue + return peak + + +def endphysics_gap(stdout: Path) -> tuple[float | None, int | None]: + release_times: dict[int, dict[int, datetime]] = {} + pattern = re.compile( + r"^\[(\d{4}\.\d{2}\.\d{2}-\d{2}\.\d{2}\.\d{2}:\d{3})\].*" + r"LogTick: tick\s+(\d+)\s+-+\s+Release tick group ([34])") + with stdout.open(encoding="utf-8", errors="replace") as stream: + for line in stream: + match = pattern.match(line) + if not match: + continue + timestamp = datetime.strptime(match.group(1), "%Y.%m.%d-%H.%M.%S:%f") + frame = int(match.group(2)) + group = int(match.group(3)) + release_times.setdefault(frame, {})[group] = timestamp + gaps = [(round((times[4] - times[3]).total_seconds() * 1000.0, 3), frame) + for frame, times in release_times.items() if 3 in times and 4 in times] + return max(gaps, default=(None, None)) + + +def run(stratum: str, repeat: int, phase: str, ready_depth: int, + background_cap: int, worker_reserve: int, helper_background: int, + tile_background_low: int, tile_background_normal: int, startup_boost: int, + tick_logs: bool, insights: bool) -> dict: + label = (f"{phase}-{stratum.lower()}-rd{ready_depth}-bg{background_cap}" + f"-reserve{worker_reserve}-helperbg{helper_background}" + f"-tilelow{tile_background_low}-tilenormal{tile_background_normal}" + f"-ticklog{int(tick_logs)}" + f"-boost{startup_boost}-{repeat}") + trace = OUT / f"{label}.trace.jsonl" + utrace = OUT / f"{label}.utrace" + abslog = OUT / f"{label}.log" + stdout = OUT / f"{label}.stdout.log" + output = HOST_PLUGIN_SAVED / "output" / label + userdir = HOST_SAVED / "user" / label + zendata = HOST_SAVED / "zen" / label + ddc = HOST_SAVED / "DDC" + for directory in (OUT, output, userdir, zendata, ddc): + directory.mkdir(parents=True, exist_ok=True) + trace.unlink(missing_ok=True) + utrace.unlink(missing_ok=True) + spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWNS_M[stratum]) + command = [ + str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended", + "-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch", + "-notraceserver", "-nocrashreports", + "-DisablePlugins=AndroidFileServer", + "-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}", + f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0", + "-archetype=TunnelNetwork", f"-slot={1 if stratum == 'FI' else 0}", + "-modes=export", "-gameconfig=1", "-coreworldtest=1", + "-ownerfourstrateplan=1", "-corevoxelsizecm=25", "-coreislandtest=0", + "-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0", + "-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0", + f"-out={output}", "-voxel.StreamingCoarseFirst=1", + "-voxel.StreamingDynamicNearReserve=1", + "-voxel.StreamingLegacyPendingResultCap=0", + "-voxel.CoreRegionConcurrentAdmission=1", + "-voxel.StreamingResultBacklogCap=256", + f"-voxel.StreamingReadyQueueDepth={ready_depth}", + f"-voxel.StreamingBackgroundWorkerCap={background_cap}", + f"-voxel.StreamingBackgroundWorkerReserve={worker_reserve}", + f"-voxel.CoreParallelForBackgroundPriority={helper_background}", + f"-voxel.StreamingTileBackgroundLow={tile_background_low}", + f"-voxel.StreamingTileBackgroundNormal={tile_background_normal}", + f"-voxel.TestLogTickFunctions={int(tick_logs)}", + "-voxel.CoreColumnCacheDirectMapped=0", "-voxel.CoreMCGradientMemoization=1", + "-voxel.StreamingCollisionLookahead=1", + f"-voxel.StreamingStartupApplyBoost={startup_boost}", + "-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=140.0", + f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}", + f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}", + f"-voxel.TestExitSeconds={EXIT_S:.1f}", "-voxel.TestMoveDirection=1,0,0", + "-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0", + "-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}", + "-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1", + "-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}", + f"-abslog={abslog}", + ] + if insights: + command += ["-trace=cpu,task,frame", "-statnamedevents", + f"-tracefile={utrace}", "-tracefiletrunc"] + wrapped = ["taskset", "-c", "0-11", "timeout", f"{RUN_TIMEOUT_S}s", *command] + started = time.monotonic() + with stdout.open("w", encoding="utf-8") as stream: + process = subprocess.Popen(wrapped, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT) + peak_rss = None + while process.poll() is None: + observed = sample_unreal_peak_rss() + if observed and (peak_rss is None or observed[0] > peak_rss[0]): + peak_rss = observed + time.sleep(0.25) + exit_code = process.wait() + observed = sample_unreal_peak_rss() + if observed and (peak_rss is None or observed[0] > peak_rss[0]): + peak_rss = observed + + events = read_events(trace) + worker_backend_config = next( + (e for e in events if e.get("type") == "worker_backend_config"), {}) + worker_cap_readback = next( + (e for e in events if e.get("type") == "streaming_worker_cap_readback"), {}) + samples = [e for e in events if e.get("type") == "streaming_sample"] + arrival = [e for e in samples if float(e.get("t_s", 0.0)) < MOVE_START_S] + full = next((e for e in arrival + if sum(e.get("desired_by_lod", [0] * 9)[:3]) > 0 + and sum(e.get("settled_by_lod", [0] * 9)[:3]) + >= sum(e.get("desired_by_lod", [0] * 9)[:3])), None) + first_admission = next((e for e in events if e.get("type") == "scheduler_admission" + and e.get("desired_rebuilt") == 1), None) + request_events = [e for e in events if e.get("type") == "request"] + initial_request_events = [e for e in request_events if e.get("epoch") == 0] + tile_by_ordinal = {e.get("request_ordinal"): e for e in events + if e.get("type") == "tile"} + collision_count = sum(1 for e in events if e.get("type") == "collision_ready" + and e.get("level", 9) <= 2 + and full and e.get("t_s", 1e30) <= full["t_s"]) + initial_tiles = [tile_by_ordinal[r.get("request_ordinal")] for r in initial_request_events + if full and r.get("t_s", 1e30) <= full["t_s"] + and r.get("request_ordinal") in tile_by_ordinal + and tile_by_ordinal[r["request_ordinal"]].get("level", 9) <= 2] + burst_start_s = float(first_admission["t_s"]) if first_admission else -1.0 + burst_end_s = float(full["t_s"]) if full else -1.0 + work_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "game_thread_work_frame"} + work_values = [e for e in work_by_frame.values() + if burst_start_s <= float(e.get("t_s", -1.0)) <= burst_end_s] + arrival_frame_ids = {int(e.get("frame", -1)) for e in work_values} + scheduler_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "scheduler_admission"} + scheduler_values = [e for e in events if e.get("type") == "scheduler_admission" + and burst_start_s <= float(e.get("t_s", -1.0)) <= burst_end_s] + apply_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "apply_frame"} + commandlet_tick_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "commandlet_world_tick_frame"} + garbage_collect_by_frame = {int(e.get("frame", -1)): e for e in events + if e.get("type") == "garbage_collect"} + # apply_frame N+1 records the interval containing AVoxelWorld::Tick N. Its apply_ms + # belongs to Tick N+1, so use apply_by_frame[N] for the same-tick apply breakdown. + near_intervals = [apply_by_frame[frame + 1] for frame in arrival_frame_ids + if frame + 1 in apply_by_frame] + rmc_by_frame: dict[int, dict[str, float]] = {} + for event in events: + frame = int(event.get("frame", -1)) + if event.get("type") != "rmc_submit" or frame not in arrival_frame_ids: + continue + values = rmc_by_frame.setdefault(frame, { + "component_acquire_ms": 0.0, + "create_section_group_ms": 0.0, + "create_sections_ms": 0.0, + "collision_handoff_ms": 0.0, + }) + for key in values: + values[key] += float(event.get(key, 0.0)) + collision_completion_by_frame: dict[int, float] = {} + for event in events: + frame = int(event.get("frame", -1)) + if event.get("type") == "collision_completion" and frame in arrival_frame_ids: + collision_completion_by_frame[frame] = collision_completion_by_frame.get(frame, 0.0) \ + + float(event.get("handler_ms", 0.0)) + near_requests = [r for r in request_events if full and r.get("t_s", 1e30) <= full["t_s"] + and r.get("level", 9) <= 2] + initial_lod0_counts = Counter( + tuple(r.get("tile", ())) for r in near_requests if r.get("level") == 0) + repeated_lod0_keys = [ + {"tile": list(tile), "requests": count} + for tile, count in sorted(initial_lod0_counts.items()) if count > 1] + initial_scheduler_frames = [e for e in events + if e.get("type") == "scheduler_admission" and full + and e.get("t_s", 1e30) <= full["t_s"]] + worst_interval = max(near_intervals, key=lambda e: float(e.get("frame_ms", 0.0)), default=None) + worst_interval_frame_id = int(worst_interval.get("frame", -1)) if worst_interval else -1 + worst_work_frame_id = worst_interval_frame_id - 1 + worst_work = work_by_frame.get(worst_work_frame_id, {}) + worst_apply = apply_by_frame.get(worst_work_frame_id, {}) + worst_scheduler = scheduler_by_frame.get(worst_work_frame_id, {}) + worst_rmc = rmc_by_frame.get(worst_work_frame_id, {}) + worst_collision_completion_ms = collision_completion_by_frame.get(worst_work_frame_id, 0.0) + rmc_values = [rmc_by_frame.get(frame, {}) for frame in arrival_frame_ids] + apply_values = [apply_by_frame[frame] for frame in arrival_frame_ids + if frame in apply_by_frame] + commandlet_tick_values = [commandlet_tick_by_frame[frame] for frame in arrival_frame_ids + if frame in commandlet_tick_by_frame] + garbage_collect_values = [garbage_collect_by_frame[frame] for frame in arrival_frame_ids + if frame in garbage_collect_by_frame] + + def work_stat(name: str, fraction: float | None = None) -> float | None: + values = [float(e.get(name, 0.0)) for e in work_values] + return percentile(values, fraction) if fraction is not None else max(values, default=0.0) + + def scheduler_stat(name: str, fraction: float | None = None) -> float | None: + values = [float(e.get(name, 0.0)) for e in scheduler_values] + return percentile(values, fraction) if fraction is not None else max(values, default=0.0) + + def rmc_stat(name: str, fraction: float | None = None) -> float | None: + values = [float(e.get(name, 0.0)) for e in rmc_values] + return percentile(values, fraction) if fraction is not None else max(values, default=0.0) + + def commandlet_stat(name: str, fraction: float | None = None) -> float | None: + values = [float(e.get(name, 0.0)) for e in commandlet_tick_values] + return percentile(values, fraction) if fraction is not None else max(values, default=0.0) + + row = { + "label": label, "stratum": stratum, "repeat": repeat, "phase": phase, + "ready_queue_depth": ready_depth, + "requested_background_worker_cap": background_cap, + "background_worker_cap": worker_cap_readback.get( + "streaming_background_worker_cap", background_cap), + "requested_background_worker_reserve": worker_reserve, + "background_worker_reserve": worker_backend_config.get( + "streaming_background_worker_reserve", worker_cap_readback.get( + "background_worker_reserve", worker_reserve)), + "requested_core_parallel_for_background_priority": helper_background, + "core_parallel_for_background_priority": worker_backend_config.get( + "core_parallel_for_background_priority", helper_background), + "requested_streaming_tile_background_low": tile_background_low, + "streaming_tile_background_low": worker_backend_config.get( + "streaming_tile_background_low", tile_background_low), + "requested_streaming_tile_background_normal": tile_background_normal, + "streaming_tile_background_normal": worker_backend_config.get( + "streaming_tile_background_normal", tile_background_normal), + "effective_tile_slots": worker_backend_config.get("effective_tile_slots"), + "background_worker_threads": worker_backend_config.get("taskgraph_background_threads"), + "startup_apply_boost": startup_boost, "insights": insights, + "engine_tick_logs": tick_logs, + "exit_code": exit_code, + "launch_wrapper": "taskset -c 0-11 timeout 1740s (no address-space limit)", + "peak_rss_kib": peak_rss[0] if peak_rss else None, + "peak_rss_mib": round(peak_rss[0] / 1024.0, 1) if peak_rss else None, + "peak_rss_pid": peak_rss[1] if peak_rss else None, + "endphysics_to_postphysics_gap_ms_max": None, + "endphysics_gap_frame": None, + "wall_s": round(time.monotonic() - started, 3), + "desired_t_s": next((e.get("t_s") for e in events if e.get("type") == "desired_set"), None), + "first_admission_t_s": first_admission.get("t_s") if first_admission else None, + "lod02_full_settled_t_s": full.get("t_s") if full else None, + "near_settle_from_first_admission_s": round( + float(full["t_s"]) - float(first_admission["t_s"]), 4) + if full and first_admission else None, + "desired_by_lod": full.get("desired_by_lod", [])[:3] if full else None, + "settled_by_lod": full.get("settled_by_lod", [])[:3] if full else None, + "near_admitted_tiles": len(near_requests), + "initial_lod0_admissions": sum(initial_lod0_counts.values()), + "repeated_lod0_request_keys_before_settle": repeated_lod0_keys, + "peak_result_backlog_before_settle": max( + (int(e.get("result_backlog", 0)) for e in initial_scheduler_frames), default=0), + "last_admission_t_s": max((e.get("t_s", 0.0) for e in near_requests), default=None), + "near_tile_events": len(initial_tiles), + "near_worker_seconds": round(sum(e.get("worker_task_s", 0.0) for e in initial_tiles), 4), + "near_core_seconds": round(sum(e.get("core_generate_s", 0.0) for e in initial_tiles), 4), + "collision_ready_by_settle": collision_count, + "burst_frame_count": len(near_intervals), + "burst_frame_ms_p95": round(sorted(e.get("frame_ms", 0.0) for e in near_intervals)[ + min(len(near_intervals) - 1, int(.95 * (len(near_intervals) - 1)))], 3) + if near_intervals else None, + "burst_frame_ms_max": round(max((e.get("frame_ms", 0.0) for e in near_intervals), default=0.0), 3), + "burst_full_world_tick_ms_p99": round(commandlet_stat("world_tick_ms", .99) or 0.0, 3), + "burst_full_world_tick_ms_max": round(commandlet_stat("world_tick_ms") or 0.0, 3), + "burst_world_pre_actor_ms_p99": round(commandlet_stat("world_pre_actor_ms", .99) or 0.0, 3), + "burst_world_pre_actor_ms_max": round(commandlet_stat("world_pre_actor_ms") or 0.0, 3), + "burst_world_actor_phase_ms_p99": round(commandlet_stat("world_actor_phase_ms", .99) or 0.0, 3), + "burst_world_actor_phase_ms_max": round(commandlet_stat("world_actor_phase_ms") or 0.0, 3), + "burst_world_post_actor_ms_p99": round(commandlet_stat("world_post_actor_ms", .99) or 0.0, 3), + "burst_world_post_actor_ms_max": round(commandlet_stat("world_post_actor_ms") or 0.0, 3), + "burst_commandlet_readiness_check_ms_max": round( + commandlet_stat("readiness_check_ms") or 0.0, 3), + "burst_commandlet_sleep_ms_p99": round(commandlet_stat("sleep_ms", .99) or 0.0, 3), + "burst_commandlet_sleep_ms_max": round(commandlet_stat("sleep_ms") or 0.0, 3), + "burst_commandlet_loop_ms_max": round(commandlet_stat("loop_ms") or 0.0, 3), + "burst_gc_count": len(garbage_collect_values), + "burst_gc_duration_ms_max": round(max( + (float(e.get("duration_ms", 0.0)) for e in garbage_collect_values), default=0.0), 3), + "burst_max_meshes_per_frame": max((e.get("meshes_applied", 0) for e in apply_values), default=0), + "burst_max_apply_ms": round(max((e.get("apply_ms", 0.0) for e in apply_values), default=0.0), 3), + "burst_max_admit_per_frame": max((sum(e.get("submitted_by_lod", [0] * 9)[:3]) + for e in scheduler_values), default=0), + "burst_frame_timing_count": len(work_values), + "burst_active_generation_workers_p95": round(work_stat("active_generation_workers", .95) or 0.0, 3), + "burst_active_generation_workers_max": round(work_stat("active_generation_workers") or 0.0, 3), + "burst_active_tile_tasks_max": round(work_stat("active_tile_tasks") or 0.0, 3), + "burst_world_tick_ms_p99": round(work_stat("tick_ms", .99) or 0.0, 3), + "burst_world_tick_ms_max": round(work_stat("tick_ms") or 0.0, 3), + "burst_update_chunks_ms_p99": round(work_stat("update_chunks_ms", .99) or 0.0, 3), + "burst_update_chunks_ms_max": round(work_stat("update_chunks_ms") or 0.0, 3), + "burst_pending_drain_ms_p99": round(work_stat("pending_drain_ms", .99) or 0.0, 3), + "burst_pending_drain_ms_max": round(work_stat("pending_drain_ms") or 0.0, 3), + "burst_post_apply_callbacks_ms_p99": round(work_stat("post_apply_callbacks_ms", .99) or 0.0, 3), + "burst_post_apply_callbacks_ms_max": round(work_stat("post_apply_callbacks_ms") or 0.0, 3), + "burst_visibility_reconcile_ms_p99": round(work_stat("visibility_reconcile_ms", .99) or 0.0, 3), + "burst_visibility_reconcile_ms_max": round(work_stat("visibility_reconcile_ms") or 0.0, 3), + "burst_collision_probe_ms_max": round(work_stat("collision_probe_ms") or 0.0, 3), + "burst_visible_apply_probe_ms_max": round(work_stat("visible_apply_probe_ms") or 0.0, 3), + "burst_lod0_ready_ms_max": round(work_stat("lod0_ready_ms") or 0.0, 3), + "burst_steady_trace_ms_max": round(work_stat("steady_trace_ms") or 0.0, 3), + "burst_modification_ms_max": round(work_stat("modification_ms") or 0.0, 3), + "burst_lifecycle_audit_ms_p99": round(work_stat("lifecycle_audit_ms", .99) or 0.0, 3), + "burst_lifecycle_audit_ms_max": round(work_stat("lifecycle_audit_ms") or 0.0, 3), + "burst_collision_gate_ms_p99": round(work_stat("collision_gate_ms", .99) or 0.0, 3), + "burst_collision_gate_ms_max": round(work_stat("collision_gate_ms") or 0.0, 3), + "burst_unload_ms_p99": round(work_stat("unload_ms", .99) or 0.0, 3), + "burst_unload_ms_max": round(work_stat("unload_ms") or 0.0, 3), + "burst_trace_json_ms_p99": round(work_stat("trace_json_ms", .99) or 0.0, 3), + "burst_trace_json_ms_max": round(work_stat("trace_json_ms") or 0.0, 3), + "burst_desired_rebuild_ms_max": round(scheduler_stat("desired_rebuild_ms") or 0.0, 3), + "burst_rmc_create_group_ms_p99": round(rmc_stat("create_section_group_ms", .99) or 0.0, 3), + "burst_rmc_create_group_ms_max": round(rmc_stat("create_section_group_ms") or 0.0, 3), + "burst_rmc_collision_handoff_ms_max": round(rmc_stat("collision_handoff_ms") or 0.0, 3), + "burst_collision_completion_ms_max": round(max( + collision_completion_by_frame.values(), default=0.0), 3), + "worst_frame_breakdown": { + "frame": worst_work_frame_id, + "world_tick_frame": worst_work_frame_id, + "frame_interval_end_frame": worst_interval_frame_id, + "frame_interval_ms": round(float(worst_interval.get("frame_ms", 0.0)), 3) + if worst_interval else None, + "apply_ms": round(float(worst_apply.get("apply_ms", 0.0)), 3) + if worst_apply else None, + "result_drain_ms": round(float(worst_apply.get("drain_ms", 0.0)), 3) + if worst_apply else None, + "outside_world_tick_ms": round(max(0.0, float(worst_interval.get("frame_ms", 0.0)) + - float(worst_work.get("tick_ms", 0.0))), 3) + if worst_interval else None, + "outside_full_world_tick_ms": round(max(0.0, float(worst_interval.get("frame_ms", 0.0)) + - float(commandlet_tick_by_frame.get(worst_work_frame_id, {}).get("world_tick_ms", 0.0))), 3) + if worst_interval else None, + "full_world_tick_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("world_tick_ms", 0.0)), 3), + "world_pre_actor_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("world_pre_actor_ms", 0.0)), 3), + "world_actor_phase_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("world_actor_phase_ms", 0.0)), 3), + "world_post_actor_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("world_post_actor_ms", 0.0)), 3), + "commandlet_readiness_check_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("readiness_check_ms", 0.0)), 3), + "commandlet_sleep_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("sleep_ms", 0.0)), 3), + "commandlet_loop_ms": round(float(commandlet_tick_by_frame.get( + worst_work_frame_id, {}).get("loop_ms", 0.0)), 3), + "gc_duration_ms": round(float(garbage_collect_by_frame.get( + worst_work_frame_id, {}).get("duration_ms", 0.0)), 3), + "world_tick_ms": round(float(worst_work.get("tick_ms", 0.0)), 3), + "active_generation_workers": int(worst_work.get("active_generation_workers", 0)), + "active_tile_tasks": int(worst_work.get("active_tile_tasks", 0)), + "collision_gate_ms": round(float(worst_work.get("collision_gate_ms", 0.0)), 3), + "update_chunks_ms": round(float(worst_work.get("update_chunks_ms", 0.0)), 3), + "desired_rebuild_ms": round(float(worst_scheduler.get("desired_rebuild_ms", 0.0)), 3), + "pending_drain_ms": round(float(worst_work.get("pending_drain_ms", 0.0)), 3), + "post_apply_callbacks_ms": round(float(worst_work.get("post_apply_callbacks_ms", 0.0)), 3), + "collision_probe_ms": round(float(worst_work.get("collision_probe_ms", 0.0)), 3), + "visibility_reconcile_ms": round(float(worst_work.get("visibility_reconcile_ms", 0.0)), 3), + "visible_apply_probe_ms": round(float(worst_work.get("visible_apply_probe_ms", 0.0)), 3), + "lod0_ready_ms": round(float(worst_work.get("lod0_ready_ms", 0.0)), 3), + "unload_ms": round(float(worst_work.get("unload_ms", 0.0)), 3), + "steady_trace_ms": round(float(worst_work.get("steady_trace_ms", 0.0)), 3), + "modification_ms": round(float(worst_work.get("modification_ms", 0.0)), 3), + "lifecycle_audit_ms": round(float(worst_work.get("lifecycle_audit_ms", 0.0)), 3), + "trace_json_ms": round(float(worst_work.get("trace_json_ms", 0.0)), 3), + "trace_archive_serialize_ms": round(float( + worst_work.get("trace_archive_serialize_ms", 0.0)), 3), + "collision_completion_ms": round(worst_collision_completion_ms, 3), + **{key: round(float(worst_rmc.get(key, 0.0)), 3) + for key in ("component_acquire_ms", "create_section_group_ms", + "create_sections_ms", "collision_handoff_ms")}, + }, + "trace": str(trace.relative_to(ROOT)), "utrace": str(utrace.relative_to(ROOT)), + "abslog": str(abslog.relative_to(ROOT)), "stdout": str(stdout.relative_to(ROOT)), + } + stdout_text = stdout.read_text(encoding="utf-8", errors="replace") + gap_ms, gap_frame = endphysics_gap(stdout) + row["endphysics_to_postphysics_gap_ms_max"] = gap_ms + row["endphysics_gap_frame"] = gap_frame + failure = re.search(r"\[VoxelForgeCoreWorldTest\] failed: ([^\r\n]+)", stdout_text) + if failure: + row["world_test_failure"] = failure.group(1) + with (OUT / "runs.jsonl").open("a", encoding="utf-8") as stream: + stream.write(json.dumps(row, sort_keys=True) + "\n") + (OUT / f"{label}.summary.json").write_text(json.dumps(row, indent=2) + "\n", + encoding="utf-8") + print("[L105UnrealArrival] " + json.dumps(row, sort_keys=True), flush=True) + if exit_code: + raise SystemExit(f"Unreal commandlet failed: {stdout}; exit={exit_code}") + if not full or len(arrival) < 8: + raise SystemExit(f"Unreal arrival trace incomplete: {stdout}; exit={exit_code}") + if insights and (not utrace.is_file() or utrace.stat().st_size < 1024): + raise SystemExit(f"Unreal Insights trace missing or empty: {utrace}") + return row + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--stratum", choices=("FI", "SW"), required=True) + parser.add_argument("--repeat", type=int, default=1) + parser.add_argument("--phase", choices=("probe", "before", "after", "snapshot"), default="probe") + parser.add_argument("--ready-depth", type=int, default=64) + parser.add_argument("--background-cap", type=int, choices=(0, 1), default=1) + parser.add_argument("--worker-reserve", type=int, choices=(0, 1, 2, 3), default=0) + parser.add_argument("--helper-background", type=int, choices=(0, 1), default=1) + parser.add_argument("--tile-background-low", type=int, choices=(0, 1), default=1) + parser.add_argument("--tile-background-normal", type=int, choices=(0, 1), default=0) + parser.add_argument("--startup-boost", type=int, choices=(0, 1), default=1) + parser.add_argument("--tick-logs", action="store_true") + parser.add_argument("--insights", action="store_true") + args = parser.parse_args() + if not EDITOR.is_file(): + raise SystemExit(f"missing Unreal commandlet executable: {EDITOR}") + run(args.stratum, args.repeat, args.phase, args.ready_depth, + args.background_cap, args.worker_reserve, args.helper_background, + args.tile_background_low, args.tile_background_normal, args.startup_boost, + args.tick_logs, args.insights) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())