far shells: 3D cell bins for HasDownwardHit/HasUpwardHit candidates (L104, exact)
Near tiles legitimately need floor/ceiling boundary patches, but every hit query scanned the tile's whole XY column. Direction-specific 3D cell bins conservatively index triangle bounds; the existing predicates still decide every hit. Candidate visits per query -88..-99%; identical query and hit counts. Hit-function cost ~1.7-1.9 -> ~0.35 s per FI/TN run. Near-ready @W12: FI 1.63 -> 1.52, TN 1.51 -> 1.38, VS 1.00 -> 0.92 s. All tile/whole hashes equal L103 (Clang/GCC W1/W8/W12). The 24-28% "idle" is in the serial base-shape (~0.24 s) and far-parent (~0.13 s) phases; the LOD0-1 drain is >= 99.4% busy. Fast tier PASS; Linux UE builds; FI Unreal arrival 1.96-1.98 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016dhxM43dnWvUu2B5Ph1iG1
This commit is contained in:
Fr0zkaandClaude Opus 5.5 committed
1 parent
8d62ba57ed
commit
6a80cb0ab3
15 files changed
+2009
-15
No files matched your search
@@ -6,7 +6,10 @@
|
|||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <array>
|
#include <array>
|
||||||
|
#include <bit>
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
|
#include <cstdio>
|
||||||
|
#include <cstdlib>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
@@ -823,11 +826,18 @@ inline void AppendFarConnectorShells(const FStandaloneShapeWorld& Shape,const FV
|
|||||||
struct FFarMeshColumnIndex
|
struct FFarMeshColumnIndex
|
||||||
{
|
{
|
||||||
FFarMeshColumnIndex(const FVoxelMeshData& InMesh,const FIntVector& InOrigin,int32 InExtent,
|
FFarMeshColumnIndex(const FVoxelMeshData& InMesh,const FIntVector& InOrigin,int32 InExtent,
|
||||||
int32 InCells,double InStep,double InVoxelSizeCm)
|
int32 InCells,double InStep,double InVoxelSizeCm,int32 InLevel)
|
||||||
:Mesh(InMesh),Origin(InOrigin),Extent(InExtent),Cells(InCells),Step(InStep),
|
:Mesh(InMesh),Origin(InOrigin),Extent(InExtent),Cells(InCells),Step(InStep),
|
||||||
VoxelSizeCm(InVoxelSizeCm),
|
VoxelSizeCm(InVoxelSizeCm),
|
||||||
ByColumn(static_cast<size_t>(InCells)*InCells)
|
Level(InLevel),ByColumn(static_cast<size_t>(InCells)*InCells),
|
||||||
|
bUseHitIndex(UseHitIndex()),bMeasureHitStats(MeasureHitStats())
|
||||||
{
|
{
|
||||||
|
if(bUseHitIndex)
|
||||||
|
{
|
||||||
|
const size_t CellCount=static_cast<size_t>(Cells)*Cells*Cells;
|
||||||
|
ByDownwardCell.resize(CellCount);
|
||||||
|
ByUpwardCell.resize(CellCount);
|
||||||
|
}
|
||||||
for(int32 Triangle=0;Triangle+2<Mesh.Triangles.Num();Triangle+=3)
|
for(int32 Triangle=0;Triangle+2<Mesh.Triangles.Num();Triangle+=3)
|
||||||
{
|
{
|
||||||
const int32 A=Mesh.Triangles[Triangle],B=Mesh.Triangles[Triangle+1],C=Mesh.Triangles[Triangle+2];
|
const int32 A=Mesh.Triangles[Triangle],B=Mesh.Triangles[Triangle+1],C=Mesh.Triangles[Triangle+2];
|
||||||
@@ -836,14 +846,46 @@ struct FFarMeshColumnIndex
|
|||||||
const double MaxX=std::max({PositionAxis(A,0),PositionAxis(B,0),PositionAxis(C,0)});
|
const double MaxX=std::max({PositionAxis(A,0),PositionAxis(B,0),PositionAxis(C,0)});
|
||||||
const double MinY=std::min({PositionAxis(A,1),PositionAxis(B,1),PositionAxis(C,1)});
|
const double MinY=std::min({PositionAxis(A,1),PositionAxis(B,1),PositionAxis(C,1)});
|
||||||
const double MaxY=std::max({PositionAxis(A,1),PositionAxis(B,1),PositionAxis(C,1)});
|
const double MaxY=std::max({PositionAxis(A,1),PositionAxis(B,1),PositionAxis(C,1)});
|
||||||
|
const double MinZ=std::min({PositionAxis(A,2),PositionAxis(B,2),PositionAxis(C,2)});
|
||||||
|
const double MaxZ=std::max({PositionAxis(A,2),PositionAxis(B,2),PositionAxis(C,2)});
|
||||||
const int32 X0=std::clamp(static_cast<int32>(std::floor((MinX-Origin.X)/Step)),0,Cells-1);
|
const int32 X0=std::clamp(static_cast<int32>(std::floor((MinX-Origin.X)/Step)),0,Cells-1);
|
||||||
const int32 X1=std::clamp(static_cast<int32>(std::floor((MaxX-Origin.X)/Step)),0,Cells-1);
|
const int32 X1=std::clamp(static_cast<int32>(std::floor((MaxX-Origin.X)/Step)),0,Cells-1);
|
||||||
const int32 Y0=std::clamp(static_cast<int32>(std::floor((MinY-Origin.Y)/Step)),0,Cells-1);
|
const int32 Y0=std::clamp(static_cast<int32>(std::floor((MinY-Origin.Y)/Step)),0,Cells-1);
|
||||||
const int32 Y1=std::clamp(static_cast<int32>(std::floor((MaxY-Origin.Y)/Step)),0,Cells-1);
|
const int32 Y1=std::clamp(static_cast<int32>(std::floor((MaxY-Origin.Y)/Step)),0,Cells-1);
|
||||||
for(int32 Y=Y0;Y<=Y1;++Y)for(int32 X=X0;X<=X1;++X)
|
for(int32 Y=Y0;Y<=Y1;++Y)for(int32 X=X0;X<=X1;++X)
|
||||||
ByColumn[static_cast<size_t>(Y)*Cells+X].push_back(Triangle/3);
|
ByColumn[static_cast<size_t>(Y)*Cells+X].push_back(Triangle/3);
|
||||||
|
if(!bUseHitIndex||A>=Mesh.Normals.Num()||B>=Mesh.Normals.Num()
|
||||||
|
||C>=Mesh.Normals.Num())continue;
|
||||||
|
const double NormalSum=Mesh.Normals[A].Z+Mesh.Normals[B].Z+Mesh.Normals[C].Z;
|
||||||
|
const bool bDown=!(NormalSum>=-1.5),bUp=!(NormalSum<=1.5);
|
||||||
|
if(!bDown&&!bUp)continue;
|
||||||
|
const double ZMargin=(MaxZ-MinZ)*2.0e-7+1.0e-8;
|
||||||
|
const int32 Z0=std::clamp(static_cast<int32>(std::floor((MinZ-ZMargin-Origin.Z)/Step)),0,Cells-1);
|
||||||
|
const int32 Z1=std::clamp(static_cast<int32>(std::floor((MaxZ+ZMargin-Origin.Z)/Step)),0,Cells-1);
|
||||||
|
for(int32 Z=Z0;Z<=Z1;++Z)for(int32 Y=Y0;Y<=Y1;++Y)for(int32 X=X0;X<=X1;++X)
|
||||||
|
{
|
||||||
|
const size_t Cell=(static_cast<size_t>(Z)*Cells+Y)*Cells+X;
|
||||||
|
if(bDown)ByDownwardCell[Cell].push_back(Triangle/3);
|
||||||
|
if(bUp)ByUpwardCell[Cell].push_back(Triangle/3);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
~FFarMeshColumnIndex()
|
||||||
|
{
|
||||||
|
if(!bMeasureHitStats||DownQueries+UpQueries==0)return;
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"[FarHitQueryStats] lod=%d origin=(%d,%d,%d) "
|
||||||
|
"down_queries=%llu down_candidate_tests=%llu down_hit_queries=%llu "
|
||||||
|
"up_queries=%llu up_candidate_tests=%llu up_hit_queries=%llu\n",
|
||||||
|
Level,Origin.X,Origin.Y,Origin.Z,
|
||||||
|
static_cast<unsigned long long>(DownQueries),
|
||||||
|
static_cast<unsigned long long>(DownCandidateTests),
|
||||||
|
static_cast<unsigned long long>(DownHitQueries),
|
||||||
|
static_cast<unsigned long long>(UpQueries),
|
||||||
|
static_cast<unsigned long long>(UpCandidateTests),
|
||||||
|
static_cast<unsigned long long>(UpHitQueries));
|
||||||
|
}
|
||||||
|
|
||||||
std::vector<double> VerticalHits(double X,double Y,double ZMin,double ZMax) const
|
std::vector<double> VerticalHits(double X,double Y,double ZMin,double ZMax) const
|
||||||
{
|
{
|
||||||
@@ -873,11 +915,13 @@ struct FFarMeshColumnIndex
|
|||||||
|
|
||||||
bool HasDownwardHit(double X,double Y,double Z,double Tolerance) const
|
bool HasDownwardHit(double X,double Y,double Z,double Tolerance) const
|
||||||
{
|
{
|
||||||
const int32 CX=std::clamp(static_cast<int32>(std::floor((X-Origin.X)/Step)),0,Cells-1);
|
if(bMeasureHitStats)++DownQueries;
|
||||||
const int32 CY=std::clamp(static_cast<int32>(std::floor((Y-Origin.Y)/Step)),0,Cells-1);
|
std::vector<int32> CandidateScratch;
|
||||||
const auto& Candidates=ByColumn[static_cast<size_t>(CY)*Cells+CX];
|
const auto& Candidates=FindHitCandidates(X,Y,Z,Tolerance,true,CandidateScratch);
|
||||||
|
bool bHit=false;
|
||||||
for(const int32 Triangle:Candidates)
|
for(const int32 Triangle:Candidates)
|
||||||
{
|
{
|
||||||
|
if(bMeasureHitStats)++DownCandidateTests;
|
||||||
const int32 IA=Mesh.Triangles[Triangle*3],IB=Mesh.Triangles[Triangle*3+1],
|
const int32 IA=Mesh.Triangles[Triangle*3],IB=Mesh.Triangles[Triangle*3+1],
|
||||||
IC=Mesh.Triangles[Triangle*3+2];
|
IC=Mesh.Triangles[Triangle*3+2];
|
||||||
if(IA<0||IB<0||IC<0||IA>=Mesh.Normals.Num()||IB>=Mesh.Normals.Num()
|
if(IA<0||IB<0||IC<0||IA>=Mesh.Normals.Num()||IB>=Mesh.Normals.Num()
|
||||||
@@ -893,9 +937,10 @@ struct FFarMeshColumnIndex
|
|||||||
const double W=1.0-U-V;
|
const double W=1.0-U-V;
|
||||||
if(U<-1.0e-7||V<-1.0e-7||W<-1.0e-7)continue;
|
if(U<-1.0e-7||V<-1.0e-7||W<-1.0e-7)continue;
|
||||||
const double Hit=AZ+U*(BZ-AZ)+V*(CZ-AZ);
|
const double Hit=AZ+U*(BZ-AZ)+V*(CZ-AZ);
|
||||||
if(std::abs(Hit-Z)<=Tolerance)return true;
|
if(std::abs(Hit-Z)<=Tolerance){bHit=true;break;}
|
||||||
}
|
}
|
||||||
return false;
|
if(bMeasureHitStats&&bHit)++DownHitQueries;
|
||||||
|
return bHit;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool HasDownwardCellCoverage(double X0,double Y0,double X1,double Y1,
|
bool HasDownwardCellCoverage(double X0,double Y0,double X1,double Y1,
|
||||||
@@ -917,11 +962,13 @@ struct FFarMeshColumnIndex
|
|||||||
|
|
||||||
bool HasUpwardHit(double X,double Y,double Z,double Tolerance) const
|
bool HasUpwardHit(double X,double Y,double Z,double Tolerance) const
|
||||||
{
|
{
|
||||||
const int32 CX=std::clamp(static_cast<int32>(std::floor((X-Origin.X)/Step)),0,Cells-1);
|
if(bMeasureHitStats)++UpQueries;
|
||||||
const int32 CY=std::clamp(static_cast<int32>(std::floor((Y-Origin.Y)/Step)),0,Cells-1);
|
std::vector<int32> CandidateScratch;
|
||||||
const auto& Candidates=ByColumn[static_cast<size_t>(CY)*Cells+CX];
|
const auto& Candidates=FindHitCandidates(X,Y,Z,Tolerance,false,CandidateScratch);
|
||||||
|
bool bHit=false;
|
||||||
for(const int32 Triangle:Candidates)
|
for(const int32 Triangle:Candidates)
|
||||||
{
|
{
|
||||||
|
if(bMeasureHitStats)++UpCandidateTests;
|
||||||
const int32 IA=Mesh.Triangles[Triangle*3],IB=Mesh.Triangles[Triangle*3+1],
|
const int32 IA=Mesh.Triangles[Triangle*3],IB=Mesh.Triangles[Triangle*3+1],
|
||||||
IC=Mesh.Triangles[Triangle*3+2];
|
IC=Mesh.Triangles[Triangle*3+2];
|
||||||
if(IA<0||IB<0||IC<0||IA>=Mesh.Normals.Num()||IB>=Mesh.Normals.Num()
|
if(IA<0||IB<0||IC<0||IA>=Mesh.Normals.Num()||IB>=Mesh.Normals.Num()
|
||||||
@@ -937,9 +984,10 @@ struct FFarMeshColumnIndex
|
|||||||
const double W=1.0-U-V;
|
const double W=1.0-U-V;
|
||||||
if(U<-1.0e-7||V<-1.0e-7||W<-1.0e-7)continue;
|
if(U<-1.0e-7||V<-1.0e-7||W<-1.0e-7)continue;
|
||||||
const double Hit=AZ+U*(BZ-AZ)+V*(CZ-AZ);
|
const double Hit=AZ+U*(BZ-AZ)+V*(CZ-AZ);
|
||||||
if(std::abs(Hit-Z)<=Tolerance)return true;
|
if(std::abs(Hit-Z)<=Tolerance){bHit=true;break;}
|
||||||
}
|
}
|
||||||
return false;
|
if(bMeasureHitStats&&bHit)++UpHitQueries;
|
||||||
|
return bHit;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool HasUpwardCellCoverage(double X0,double Y0,double X1,double Y1,
|
bool HasUpwardCellCoverage(double X0,double Y0,double X1,double Y1,
|
||||||
@@ -952,6 +1000,53 @@ struct FFarMeshColumnIndex
|
|||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
const std::vector<int32>& FindHitCandidates(double X,double Y,double Z,double Tolerance,
|
||||||
|
bool bDown,std::vector<int32>& Scratch) const
|
||||||
|
{
|
||||||
|
const int32 CX=std::clamp(static_cast<int32>(std::floor((X-Origin.X)/Step)),0,Cells-1);
|
||||||
|
const int32 CY=std::clamp(static_cast<int32>(std::floor((Y-Origin.Y)/Step)),0,Cells-1);
|
||||||
|
const size_t Column=static_cast<size_t>(CY)*Cells+CX;
|
||||||
|
if(!bUseHitIndex||!std::isfinite(Tolerance)||Tolerance<0.0||Step<=0.0)
|
||||||
|
return ByColumn[Column];
|
||||||
|
const int32 Z0=std::clamp(static_cast<int32>(std::floor((Z-Tolerance-Origin.Z)/Step)),
|
||||||
|
0,Cells-1);
|
||||||
|
const int32 Z1=std::clamp(static_cast<int32>(std::floor((Z+Tolerance-Origin.Z)/Step)),
|
||||||
|
0,Cells-1);
|
||||||
|
const auto& Buckets=bDown?ByDownwardCell:ByUpwardCell;
|
||||||
|
const size_t First=(static_cast<size_t>(Z0)*Cells+CY)*Cells+CX;
|
||||||
|
if(Z0==Z1)return Buckets[First];
|
||||||
|
Scratch.clear();
|
||||||
|
for(int32 BucketZ=Z0;BucketZ<=Z1;++BucketZ)
|
||||||
|
{
|
||||||
|
const size_t Cell=(static_cast<size_t>(BucketZ)*Cells+CY)*Cells+CX;
|
||||||
|
const auto& Candidates=Buckets[Cell];
|
||||||
|
Scratch.insert(Scratch.end(),Candidates.begin(),Candidates.end());
|
||||||
|
}
|
||||||
|
std::sort(Scratch.begin(),Scratch.end());
|
||||||
|
Scratch.erase(std::unique(Scratch.begin(),Scratch.end()),Scratch.end());
|
||||||
|
return Scratch;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool UseHitIndex()
|
||||||
|
{
|
||||||
|
static const bool Enabled=[]()
|
||||||
|
{
|
||||||
|
const char* Value=std::getenv("VOXELFORGE_L104_FAR_HIT_CACHE");
|
||||||
|
return Value==nullptr||Value[0]!='0'||Value[1]!='\0';
|
||||||
|
}();
|
||||||
|
return Enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool MeasureHitStats()
|
||||||
|
{
|
||||||
|
static const bool Enabled=[]()
|
||||||
|
{
|
||||||
|
const char* Value=std::getenv("VOXELFORGE_L104_FAR_HIT_STATS");
|
||||||
|
return Value&&Value[0]=='1'&&Value[1]=='\0';
|
||||||
|
}();
|
||||||
|
return Enabled;
|
||||||
|
}
|
||||||
|
|
||||||
double PositionAxis(int32 Index,int32 Axis) const
|
double PositionAxis(int32 Index,int32 Axis) const
|
||||||
{
|
{
|
||||||
const FVector& P=Mesh.Vertices[Index];
|
const FVector& P=Mesh.Vertices[Index];
|
||||||
@@ -963,7 +1058,12 @@ private:
|
|||||||
int32 Extent=0,Cells=0;
|
int32 Extent=0,Cells=0;
|
||||||
double Step=1.0;
|
double Step=1.0;
|
||||||
double VoxelSizeCm=25.0;
|
double VoxelSizeCm=25.0;
|
||||||
|
int32 Level=0;
|
||||||
std::vector<std::vector<int32>> ByColumn;
|
std::vector<std::vector<int32>> ByColumn;
|
||||||
|
std::vector<std::vector<int32>> ByDownwardCell,ByUpwardCell;
|
||||||
|
bool bUseHitIndex=false,bMeasureHitStats=false;
|
||||||
|
mutable std::uint64_t DownQueries=0,DownCandidateTests=0,DownHitQueries=0;
|
||||||
|
mutable std::uint64_t UpQueries=0,UpCandidateTests=0,UpHitQueries=0;
|
||||||
};
|
};
|
||||||
|
|
||||||
inline std::vector<double> FindFarStratumCrossings(const FStandaloneShapeWorld& Shape,
|
inline std::vector<double> FindFarStratumCrossings(const FStandaloneShapeWorld& Shape,
|
||||||
@@ -1092,7 +1192,7 @@ inline void AppendFarStratumShells(const FStandaloneOwnerWorld& World,const FSta
|
|||||||
const FColor Color=bHasColors&&OldVertexCount>0?Mesh.Colors[0]:FColor(160,160,160,255);
|
const FColor Color=bHasColors&&OldVertexCount>0?Mesh.Colors[0]:FColor(160,160,160,255);
|
||||||
const int32 Cells=std::max(1,World.CoarseTileCells);
|
const int32 Cells=std::max(1,World.CoarseTileCells);
|
||||||
FFarMeshColumnIndex MeshIndex(Mesh,Origin,Extent,Cells,Step,
|
FFarMeshColumnIndex MeshIndex(Mesh,Origin,Extent,Cells,Step,
|
||||||
static_cast<double>(World.CoreVoxelSizeMicrons)/10000.0);
|
static_cast<double>(World.CoreVoxelSizeMicrons)/10000.0,Key.Level);
|
||||||
std::vector<FVector> NewVertices,NewNormals;
|
std::vector<FVector> NewVertices,NewNormals;
|
||||||
std::vector<FVector2f> NewUVs;std::vector<FColor> NewColors;
|
std::vector<FVector2f> NewUVs;std::vector<FColor> NewColors;
|
||||||
std::vector<int32> NewTriangles;
|
std::vector<int32> NewTriangles;
|
||||||
@@ -1374,7 +1474,7 @@ inline void AppendStratumBoundaryCeilings(const FStandaloneOwnerWorld& World,
|
|||||||
|
|
||||||
const int32 Cells=std::max(1,World.CoarseTileCells);
|
const int32 Cells=std::max(1,World.CoarseTileCells);
|
||||||
FFarMeshColumnIndex MeshIndex(Mesh,Origin,Extent,Cells,Step,
|
FFarMeshColumnIndex MeshIndex(Mesh,Origin,Extent,Cells,Step,
|
||||||
static_cast<double>(World.CoreVoxelSizeMicrons)/10000.0);
|
static_cast<double>(World.CoreVoxelSizeMicrons)/10000.0,Key.Level);
|
||||||
const int32 OldVertexCount=Mesh.Vertices.Num();
|
const int32 OldVertexCount=Mesh.Vertices.Num();
|
||||||
const int32 OldSurfaceVertices=std::clamp(Mesh.NumSurfaceVertices,0,OldVertexCount);
|
const int32 OldSurfaceVertices=std::clamp(Mesh.NumSurfaceVertices,0,OldVertexCount);
|
||||||
const bool bHasUVs=Mesh.UVs.Num()==OldVertexCount;
|
const bool bHasUVs=Mesh.UVs.Num()==OldVertexCount;
|
||||||
@@ -1550,7 +1650,7 @@ inline void AppendStratumBoundaryFloors(const FStandaloneOwnerWorld& World,
|
|||||||
|
|
||||||
const int32 Cells=std::max(1,World.CoarseTileCells);
|
const int32 Cells=std::max(1,World.CoarseTileCells);
|
||||||
FFarMeshColumnIndex MeshIndex(Mesh,Origin,Extent,Cells,Step,
|
FFarMeshColumnIndex MeshIndex(Mesh,Origin,Extent,Cells,Step,
|
||||||
static_cast<double>(World.CoreVoxelSizeMicrons)/10000.0);
|
static_cast<double>(World.CoreVoxelSizeMicrons)/10000.0,Key.Level);
|
||||||
const int32 OldVertexCount=Mesh.Vertices.Num();
|
const int32 OldVertexCount=Mesh.Vertices.Num();
|
||||||
const int32 OldSurfaceVertices=std::clamp(Mesh.NumSurfaceVertices,0,OldVertexCount);
|
const int32 OldSurfaceVertices=std::clamp(Mesh.NumSurfaceVertices,0,OldVertexCount);
|
||||||
const bool bHasUVs=Mesh.UVs.Num()==OldVertexCount;
|
const bool bHasUVs=Mesh.UVs.Num()==OldVertexCount;
|
||||||
|
|||||||
@@ -0,0 +1,344 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Close L103 sampler costs against direct grid and tile-worker timings."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import statistics
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/census"
|
||||||
|
WORKERS = 12
|
||||||
|
CALLERS = (
|
||||||
|
"unknown", "density_grid", "gradient_normal_fallback", "mc_midpoint_correction",
|
||||||
|
"skirt_surface_vertex", "skirt_extruded_vertex", "skirt_clipped_vertex",
|
||||||
|
"skirt_refinement_face", "skirt_coarse_centroid", "far_shell_surface_query",
|
||||||
|
"far_shell_projection_normal", "far_shell_stratum_refinement",
|
||||||
|
)
|
||||||
|
SKIRT_CALLERS = {
|
||||||
|
"skirt_surface_vertex": "skirt_surface_vertex_offgrid",
|
||||||
|
"skirt_extruded_vertex": "skirt_extruded_vertex_offgrid",
|
||||||
|
"skirt_clipped_vertex": "skirt_clipped_vertex_offgrid",
|
||||||
|
"skirt_refinement_face": "skirt_refinement_face_offgrid",
|
||||||
|
"skirt_coarse_centroid": "skirt_coarse_centroid_offgrid",
|
||||||
|
}
|
||||||
|
ARRAY_FIELDS = (
|
||||||
|
"sweep_missing_by_caller", "sweep_exact_fallback_by_caller",
|
||||||
|
"sweep_bound_skip_by_caller", "sweep_tile_free_skip_by_caller",
|
||||||
|
"sweep_vertical_bound_skip_by_caller", "sweep_negative_reach_skip_by_caller",
|
||||||
|
"sweep_positive_reach_skip_by_caller", "density_offgrid_float_by_caller",
|
||||||
|
"density_offgrid_unclamped_by_caller", "density_offgrid_detail_by_caller",
|
||||||
|
)
|
||||||
|
CPU_FIELDS = (
|
||||||
|
"density_float_points", "density_lattice_points", "density_unclamped_points",
|
||||||
|
"grad_calls", "grad_memo_hits", "grad_fallback_calls", "midpoint_attempts",
|
||||||
|
"midpoint_memo_hits", "skirt_density_probes", "skirt_surface_vertex",
|
||||||
|
"skirt_surface_vertex_offgrid", "skirt_extruded_vertex",
|
||||||
|
"skirt_extruded_vertex_offgrid", "skirt_clipped_vertex",
|
||||||
|
"skirt_clipped_vertex_offgrid", "skirt_refinement_face",
|
||||||
|
"skirt_refinement_face_offgrid", "skirt_coarse_centroid",
|
||||||
|
"skirt_coarse_centroid_offgrid",
|
||||||
|
)
|
||||||
|
COMPOSER_FIELDS = (
|
||||||
|
"queries", "boundary_band_queries", "column_distance_calls", "carve_segment_visits",
|
||||||
|
"support_segment_visits", "support_capsule_checks", "later_bore_checks",
|
||||||
|
"other_capsule_checks", "mouth_support_calls", "mouth_route_segment_checks",
|
||||||
|
"relief_field_calls", "sweep_segments", "sweep_pairs", "sweep_lookups",
|
||||||
|
"sweep_cache_hits", "sweep_missing", "sweep_base_samples", "sweep_detail_samples",
|
||||||
|
"sweep_grid_samples", "sweep_no_influence_samples",
|
||||||
|
)
|
||||||
|
SAMPLE = re.compile(
|
||||||
|
r"^\[L77Sample\] pc=(0x\w+) lod=(\d+) stage=(\S+) route=(\S+) "
|
||||||
|
r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M)
|
||||||
|
SAMPLER = re.compile(r"^\[L77Sampler\] action=stop (.*)$", re.M)
|
||||||
|
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||||
|
CPU_LINE = re.compile(r"^\[CpuTileStages\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M)
|
||||||
|
COMPOSER_LINE = re.compile(r"^\[PassageComposerTile\] key=\([^,]+,[^,]+,[^,]+,lod(\d+)\) (.*)$", re.M)
|
||||||
|
WORKER_LINE = re.compile(r"^\[WorldAssemblerLODWorker\] lod=(\d+) (.*)$", re.M)
|
||||||
|
DETAIL_LINE = re.compile(r"^\[WorldAssemblerLODCostDetail\] lod=(\d+) (.*)$", re.M)
|
||||||
|
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||||
|
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||||
|
|
||||||
|
|
||||||
|
def kv(payload: str) -> dict[str, str]:
|
||||||
|
return dict(FIELDS.findall(payload))
|
||||||
|
|
||||||
|
|
||||||
|
def array_value(text: str | None) -> list[int]:
|
||||||
|
values = [int(value) for value in text.split(",")] if text else []
|
||||||
|
if len(values) != len(CALLERS):
|
||||||
|
raise ValueError(f"expected {len(CALLERS)} caller values, found {len(values)}")
|
||||||
|
return values
|
||||||
|
|
||||||
|
|
||||||
|
def sum_rows(rows: list[dict], fields: tuple[str, ...], lod_max: int | None) -> dict:
|
||||||
|
selected = [row for row in rows if lod_max is None or row["lod"] <= lod_max]
|
||||||
|
result = {field: sum(row[field] for row in selected) for field in fields}
|
||||||
|
for field in ARRAY_FIELDS:
|
||||||
|
result[field] = [sum(row[field][i] for row in selected) for i in range(len(CALLERS))]
|
||||||
|
result["tiles"] = len(selected)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def analyze_run(run: dict) -> tuple[dict, list[dict], list[dict], list[dict]]:
|
||||||
|
path = ROOT / run["log"]
|
||||||
|
content = path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
sampler_match = SAMPLER.search(content)
|
||||||
|
stamp_match = STAMP.search(content)
|
||||||
|
pipeline_match = PIPELINE.search(content)
|
||||||
|
if sampler_match is None or stamp_match is None or pipeline_match is None:
|
||||||
|
raise RuntimeError(f"sampler, pipeline, or source stamp missing: {path}")
|
||||||
|
pipeline = kv(pipeline_match.group(1))
|
||||||
|
if int(pipeline.get("workers", 0)) != WORKERS:
|
||||||
|
raise RuntimeError(f"expected W{WORKERS} census run: {path}: {pipeline}")
|
||||||
|
sampler = kv(sampler_match.group(1))
|
||||||
|
if sampler.get("status") != "PASS" or int(sampler.get("lost", "-1")) != 0:
|
||||||
|
raise RuntimeError(f"sampler loss or failure: {path} {sampler}")
|
||||||
|
samples = [
|
||||||
|
{"pc": pc, "lod": int(lod), "stage": stage, "route": route,
|
||||||
|
"phase": phase, "halo": int(halo), "weight": int(weight)}
|
||||||
|
for pc, lod, stage, route, phase, halo, weight in SAMPLE.findall(content)
|
||||||
|
]
|
||||||
|
stored = int(sampler["samples"])
|
||||||
|
total_weight = sum(row["weight"] for row in samples)
|
||||||
|
expanded = int(sampler["expanded_ticks"])
|
||||||
|
if len(samples) != stored or total_weight != expanded:
|
||||||
|
raise RuntimeError(f"sample record closure failed: {path} rows={len(samples)}/{stored} "
|
||||||
|
f"weight={total_weight}/{expanded}")
|
||||||
|
scale_s = float(sampler["profiled_thread_cpu_s"]) / expanded
|
||||||
|
|
||||||
|
cpu_rows, composer_rows, worker_rows, detail_rows = [], [], [], []
|
||||||
|
for lod, tail in CPU_LINE.findall(content):
|
||||||
|
values = kv(tail)
|
||||||
|
row = {"lod": int(lod)}
|
||||||
|
row.update({name: int(values.get(name, 0)) for name in CPU_FIELDS})
|
||||||
|
row.update({name: array_value(values.get(name)) for name in ARRAY_FIELDS})
|
||||||
|
cpu_rows.append(row)
|
||||||
|
for lod, tail in COMPOSER_LINE.findall(content):
|
||||||
|
values = kv(tail)
|
||||||
|
row = {"lod": int(lod)}
|
||||||
|
row.update({name: int(values.get(name, 0)) for name in COMPOSER_FIELDS})
|
||||||
|
composer_rows.append(row)
|
||||||
|
for regex, destination, names in (
|
||||||
|
(WORKER_LINE, worker_rows, ("tiles", "worker_sum_s", "column_eval_exclusive_ms",
|
||||||
|
"mc_exclusive_ms", "ground_refine_exclusive_ms",
|
||||||
|
"cap_refine_exclusive_ms", "skirt_ms", "mesh_assembly_ms")),
|
||||||
|
(DETAIL_LINE, detail_rows, ("tiles", "mc_exclusive_ms", "mc_grid_ms",
|
||||||
|
"mc_cell_class_ms", "mc_gradient_ms", "mc_interpolation_ms",
|
||||||
|
"mc_dedup_stream_ms", "mc_other_ms", "ground_exclusive_ms",
|
||||||
|
"cap_exclusive_ms", "stratum_shell_ms", "connector_shell_ms",
|
||||||
|
"stratum_column_compute_ms", "connector_column_compute_ms")),
|
||||||
|
):
|
||||||
|
for lod, tail in regex.findall(content):
|
||||||
|
values = kv(tail)
|
||||||
|
row = {"lod": int(lod)}
|
||||||
|
for name in names:
|
||||||
|
if name == "tiles":
|
||||||
|
row[name] = int(values.get(name, 0))
|
||||||
|
else:
|
||||||
|
row[name] = float(values.get(name, 0))
|
||||||
|
destination.append(row)
|
||||||
|
if not cpu_rows or not composer_rows or not detail_rows:
|
||||||
|
raise RuntimeError(f"instrumented tile output missing: {path}")
|
||||||
|
|
||||||
|
near_samples = [row for row in samples if row["lod"] <= 2]
|
||||||
|
all_phase = Counter()
|
||||||
|
near_phase = Counter()
|
||||||
|
near_route_phase = Counter()
|
||||||
|
near_stage_phase = Counter()
|
||||||
|
near_lod_stage_phase = Counter()
|
||||||
|
for sample in samples:
|
||||||
|
all_phase[sample["phase"]] += sample["weight"]
|
||||||
|
if sample["lod"] <= 2:
|
||||||
|
near_phase[sample["phase"]] += sample["weight"]
|
||||||
|
near_route_phase[(sample["route"], sample["phase"])] += sample["weight"]
|
||||||
|
near_stage_phase[(sample["stage"], sample["phase"])] += sample["weight"]
|
||||||
|
near_lod_stage_phase[(sample["lod"], sample["stage"], sample["phase"])] += sample["weight"]
|
||||||
|
|
||||||
|
near_cpu = sum_rows(cpu_rows, CPU_FIELDS, 2)
|
||||||
|
all_cpu = sum_rows(cpu_rows, CPU_FIELDS, None)
|
||||||
|
near_composer = {field: sum(row[field] for row in composer_rows if row["lod"] <= 2)
|
||||||
|
for field in COMPOSER_FIELDS}
|
||||||
|
all_composer = {field: sum(row[field] for row in composer_rows) for field in COMPOSER_FIELDS}
|
||||||
|
near_workers = [row for row in worker_rows if row["lod"] <= 2]
|
||||||
|
near_details = [row for row in detail_rows if row["lod"] <= 2]
|
||||||
|
direct_worker_s = sum(row["worker_sum_s"] for row in near_workers)
|
||||||
|
direct_grid_s = sum(row["mc_grid_ms"] for row in near_details) / 1000.0
|
||||||
|
sample_near_cpu_s = sum(near_phase.values()) * scale_s
|
||||||
|
sample_grid_s = near_stage_phase[("grid_fill", "base_field")] * scale_s
|
||||||
|
sample_grid_allphase_s = sum(
|
||||||
|
count for (stage, _phase), count in near_stage_phase.items() if stage == "grid_fill") * scale_s
|
||||||
|
near_total_ticks = sum(near_phase.values())
|
||||||
|
near_phase_cpu_s = {phase: direct_worker_s * ticks / near_total_ticks
|
||||||
|
for phase, ticks in near_phase.items()} if near_total_ticks else {}
|
||||||
|
direct_worker_by_lod = {row["lod"]: row["worker_sum_s"] for row in near_workers}
|
||||||
|
sampled_grid_estimate_s = 0.0
|
||||||
|
for lod, worker_s in direct_worker_by_lod.items():
|
||||||
|
stage_ticks = sum(weight for (sample_lod, stage, _phase), weight
|
||||||
|
in near_lod_stage_phase.items()
|
||||||
|
if sample_lod == lod and stage == "grid_fill")
|
||||||
|
lod_ticks = sum(weight for (sample_lod, _stage, _phase), weight
|
||||||
|
in near_lod_stage_phase.items() if sample_lod == lod)
|
||||||
|
if lod_ticks:
|
||||||
|
sampled_grid_estimate_s += worker_s * stage_ticks / lod_ticks
|
||||||
|
# Compare the sample-derived per-LOD grid-fill share with direct mc_grid_ms.
|
||||||
|
grid_closure_pct = 100.0 * sampled_grid_estimate_s / direct_grid_s if direct_grid_s else 0.0
|
||||||
|
near_direct = {name: sum(row[name] for row in near_details)
|
||||||
|
for name in ("mc_exclusive_ms", "mc_grid_ms", "mc_cell_class_ms",
|
||||||
|
"mc_gradient_ms", "mc_interpolation_ms", "mc_dedup_stream_ms",
|
||||||
|
"mc_other_ms", "ground_exclusive_ms", "cap_exclusive_ms",
|
||||||
|
"stratum_shell_ms", "connector_shell_ms",
|
||||||
|
"stratum_column_compute_ms", "connector_column_compute_ms")}
|
||||||
|
all_direct = {name: sum(row[name] for row in detail_rows)
|
||||||
|
for name in ("stratum_shell_ms", "connector_shell_ms",
|
||||||
|
"stratum_column_compute_ms", "connector_column_compute_ms")}
|
||||||
|
|
||||||
|
skip_fields = ("sweep_exact_fallback_by_caller", "sweep_bound_skip_by_caller",
|
||||||
|
"sweep_tile_free_skip_by_caller", "sweep_vertical_bound_skip_by_caller",
|
||||||
|
"sweep_negative_reach_skip_by_caller", "sweep_positive_reach_skip_by_caller")
|
||||||
|
unresolved = [near_cpu["sweep_missing_by_caller"][i]
|
||||||
|
- sum(near_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))]
|
||||||
|
all_unresolved = [all_cpu["sweep_missing_by_caller"][i]
|
||||||
|
- sum(all_cpu[name][i] for name in skip_fields) for i in range(len(CALLERS))]
|
||||||
|
skirt_checks = {
|
||||||
|
caller: near_cpu["density_offgrid_float_by_caller"][CALLERS.index(caller)]
|
||||||
|
== near_cpu[probe]
|
||||||
|
for caller, probe in SKIRT_CALLERS.items()
|
||||||
|
}
|
||||||
|
validation = {
|
||||||
|
"sampler_sample_count_and_expanded_tick_sum": True,
|
||||||
|
"all_sweep_misses_match_composer": sum(all_cpu["sweep_missing_by_caller"])
|
||||||
|
== all_composer["sweep_missing"],
|
||||||
|
"all_misses_have_exact_or_skip_disposition": all(value == 0 for value in all_unresolved),
|
||||||
|
"near_misses_have_exact_or_skip_disposition": all(value == 0 for value in unresolved),
|
||||||
|
"near_midpoint_unique_matches_unclamped_offgrid":
|
||||||
|
near_cpu["midpoint_attempts"] - near_cpu["midpoint_memo_hits"]
|
||||||
|
== near_cpu["density_offgrid_unclamped_by_caller"][CALLERS.index("mc_midpoint_correction")],
|
||||||
|
"near_skirt_callers_match_offgrid_float": all(skirt_checks.values()),
|
||||||
|
"near_skirt_caller_checks": skirt_checks,
|
||||||
|
"grid_fill_closure_diagnostic_recorded": direct_grid_s > 0,
|
||||||
|
}
|
||||||
|
base_samples = near_composer["sweep_base_samples"]
|
||||||
|
detail_samples = near_composer["sweep_detail_samples"]
|
||||||
|
total_sweep_samples = base_samples + detail_samples
|
||||||
|
row = {
|
||||||
|
"stratum": run["stratum"], "replicate": run["replicate"],
|
||||||
|
"workers": WORKERS, "source_stamp": stamp_match.group(1),
|
||||||
|
"exit_code": run["exit_code"],
|
||||||
|
"sampler_samples": stored, "expanded_ticks": expanded,
|
||||||
|
"sampler_lost": int(sampler["lost"]), "timer_overruns": int(sampler["timer_overruns"]),
|
||||||
|
"timer_threads": int(sampler["timer_threads"]),
|
||||||
|
"profiled_thread_cpu_s": float(sampler["profiled_thread_cpu_s"]),
|
||||||
|
"whole_process_cpu_s": float(sampler["whole_process_cpu_s"]),
|
||||||
|
"handler_cpu_ms": float(sampler["handler_cpu_ms"]),
|
||||||
|
"handler_share_pct": float(sampler["handler_share_pct"]),
|
||||||
|
"observed_us_per_tick": float(sampler["observed_us_per_tick"]),
|
||||||
|
"near_lod_samples": len(near_samples),
|
||||||
|
"near_lod_weighted_ticks": sum(near_phase.values()),
|
||||||
|
"near_sampled_worker_cpu_s": sample_near_cpu_s,
|
||||||
|
"near_direct_worker_cpu_s": direct_worker_s,
|
||||||
|
"near_sample_to_direct_worker_pct": 100.0 * sample_near_cpu_s / direct_worker_s if direct_worker_s else 0.0,
|
||||||
|
"near_sampled_grid_fill_s": sample_grid_allphase_s,
|
||||||
|
"near_grid_fill_share_estimate_s": sampled_grid_estimate_s,
|
||||||
|
"near_phase_worker_cpu_s": near_phase_cpu_s,
|
||||||
|
"near_phase_worker_cpu_pct": {phase: 100.0 * ticks / near_total_ticks
|
||||||
|
for phase, ticks in near_phase.items()}
|
||||||
|
if near_total_ticks else {},
|
||||||
|
"near_direct_grid_fill_s": direct_grid_s,
|
||||||
|
"near_grid_fill_closure_pct": grid_closure_pct,
|
||||||
|
"near_grid_fill_base_field_samples_s": sample_grid_s,
|
||||||
|
"near_phase_ticks": dict(near_phase),
|
||||||
|
"near_route_phase_ticks": {f"{r}|{p}": count for (r, p), count in near_route_phase.items()},
|
||||||
|
"near_stage_phase_ticks": {f"{s}|{p}": count for (s, p), count in near_stage_phase.items()},
|
||||||
|
"all_phase_ticks": dict(all_phase),
|
||||||
|
"near_direct_stage_ms": near_direct,
|
||||||
|
"all_direct_far_shell_ms": all_direct,
|
||||||
|
"near_cpu_counts": near_cpu, "all_cpu_counts": all_cpu,
|
||||||
|
"near_composer_counts": near_composer, "all_composer_counts": all_composer,
|
||||||
|
"near_sweep_base_samples": base_samples, "near_sweep_detail_samples": detail_samples,
|
||||||
|
"near_sweep_base_share_pct": 100.0 * base_samples / total_sweep_samples if total_sweep_samples else 0.0,
|
||||||
|
"near_sweep_detail_share_pct": 100.0 * detail_samples / total_sweep_samples if total_sweep_samples else 0.0,
|
||||||
|
"all_sweep_base_samples": all_composer["sweep_base_samples"],
|
||||||
|
"all_sweep_detail_samples": all_composer["sweep_detail_samples"],
|
||||||
|
"validation": validation,
|
||||||
|
"run_log": run["log"],
|
||||||
|
}
|
||||||
|
phase_rows = []
|
||||||
|
near_total = sum(near_phase.values())
|
||||||
|
for phase, ticks in sorted(near_phase.items()):
|
||||||
|
phase_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
||||||
|
"phase": phase, "weighted_ticks": ticks,
|
||||||
|
"sampled_cpu_s": ticks * scale_s,
|
||||||
|
"share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0,
|
||||||
|
"direct_worker_cpu_s_estimate": near_phase_cpu_s.get(phase, 0.0),
|
||||||
|
"share_near_direct_worker_cpu_pct":
|
||||||
|
100 * ticks / near_total if near_total else 0})
|
||||||
|
route_rows = []
|
||||||
|
for (route, phase), ticks in sorted(near_route_phase.items()):
|
||||||
|
route_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
||||||
|
"route": route, "phase": phase, "weighted_ticks": ticks,
|
||||||
|
"sampled_cpu_s": ticks * scale_s,
|
||||||
|
"share_near_sampled_cpu_pct": 100 * ticks / near_total if near_total else 0,
|
||||||
|
"direct_worker_cpu_s_estimate":
|
||||||
|
direct_worker_s * ticks / near_total if near_total else 0.0,
|
||||||
|
"share_near_direct_worker_cpu_pct":
|
||||||
|
100 * ticks / near_total if near_total else 0})
|
||||||
|
caller_rows = []
|
||||||
|
for i, caller in enumerate(CALLERS):
|
||||||
|
caller_rows.append({"stratum": run["stratum"], "replicate": run["replicate"],
|
||||||
|
"caller": caller,
|
||||||
|
**{name.removesuffix("_by_caller"): near_cpu[name][i]
|
||||||
|
for name in ARRAY_FIELDS}})
|
||||||
|
return row, phase_rows, route_rows, caller_rows
|
||||||
|
|
||||||
|
|
||||||
|
def write_csv(path: Path, rows: list[dict]) -> None:
|
||||||
|
if rows:
|
||||||
|
with path.open("w", encoding="utf-8", newline="") as stream:
|
||||||
|
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
runs = json.loads((OUT / "runs.json").read_text(encoding="utf-8"))
|
||||||
|
details, phase_rows, route_rows, caller_rows = [], [], [], []
|
||||||
|
for run in runs:
|
||||||
|
detail, phases, routes, callers = analyze_run(run)
|
||||||
|
details.append(detail)
|
||||||
|
phase_rows.extend(phases)
|
||||||
|
route_rows.extend(routes)
|
||||||
|
caller_rows.extend(callers)
|
||||||
|
means = []
|
||||||
|
for stratum in ("SW", "FI", "TN", "VS"):
|
||||||
|
subset = [row for row in details if row["stratum"] == stratum]
|
||||||
|
for metric in ("near_sampled_worker_cpu_s", "near_direct_worker_cpu_s",
|
||||||
|
"near_sample_to_direct_worker_pct", "near_sampled_grid_fill_s",
|
||||||
|
"near_grid_fill_share_estimate_s", "near_direct_grid_fill_s",
|
||||||
|
"near_grid_fill_closure_pct",
|
||||||
|
"near_sweep_base_share_pct", "near_sweep_detail_share_pct"):
|
||||||
|
means.append({"stratum": stratum, "metric": metric,
|
||||||
|
"runs": len(subset), "mean": statistics.mean(row[metric] for row in subset),
|
||||||
|
"median": statistics.median(row[metric] for row in subset),
|
||||||
|
"min": min(row[metric] for row in subset),
|
||||||
|
"max": max(row[metric] for row in subset)})
|
||||||
|
write_csv(OUT / "phase-cost.csv", phase_rows)
|
||||||
|
write_csv(OUT / "route-phase-cost.csv", route_rows)
|
||||||
|
write_csv(OUT / "caller-dispositions-near.csv", caller_rows)
|
||||||
|
summary = {"runs": details, "metrics": means,
|
||||||
|
"validation_pass": all(
|
||||||
|
all(value for key, value in row["validation"].items()
|
||||||
|
if key != "near_skirt_caller_checks")
|
||||||
|
and all(row["validation"]["near_skirt_caller_checks"].values())
|
||||||
|
for row in details)}
|
||||||
|
(OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||||
|
print("[L103CensusSummary] " + json.dumps(
|
||||||
|
{"validation_pass": summary["validation_pass"], "metrics": means}, sort_keys=True), flush=True)
|
||||||
|
return 0 if summary["validation_pass"] else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Aggregate per-index far-hit probe counters by stratum, LOD and direction."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/probes"
|
||||||
|
|
||||||
|
|
||||||
|
def aggregate(run: dict) -> dict[tuple[int, str], dict[str, int]]:
|
||||||
|
result: dict[tuple[int, str], dict[str, int]] = defaultdict(
|
||||||
|
lambda: {"queries": 0, "candidate_tests": 0, "hit_queries": 0})
|
||||||
|
for index in run["stats"]:
|
||||||
|
for direction in ("down", "up"):
|
||||||
|
row = result[(index["lod"], direction)]
|
||||||
|
for target, source in (("queries", "queries"),
|
||||||
|
("candidate_tests", "candidate_tests"),
|
||||||
|
("hit_queries", "hit_queries")):
|
||||||
|
row[target] += index[f"{direction}_{source}"]
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
runs = json.loads((OUT / "runs.json").read_text(encoding="utf-8"))
|
||||||
|
lookup = {(run["stratum"], run["cache"]): aggregate(run) for run in runs}
|
||||||
|
summary = []
|
||||||
|
for stratum in ("SW", "FI", "TN", "VS"):
|
||||||
|
before, after = lookup[(stratum, "off")], lookup[(stratum, "on")]
|
||||||
|
for lod, direction in sorted(set(before) | set(after)):
|
||||||
|
a = before.get((lod, direction), {"queries": 0, "candidate_tests": 0,
|
||||||
|
"hit_queries": 0})
|
||||||
|
b = after.get((lod, direction), {"queries": 0, "candidate_tests": 0,
|
||||||
|
"hit_queries": 0})
|
||||||
|
if a["queries"] != b["queries"] or a["hit_queries"] != b["hit_queries"]:
|
||||||
|
raise RuntimeError(f"query/result drift for {stratum} LOD{lod} {direction}: {a} != {b}")
|
||||||
|
summary.append({
|
||||||
|
"stratum": stratum, "lod": lod, "direction": direction,
|
||||||
|
"queries": a["queries"], "hit_queries": a["hit_queries"],
|
||||||
|
"baseline_candidate_tests": a["candidate_tests"],
|
||||||
|
"candidate_tests_after": b["candidate_tests"],
|
||||||
|
"candidate_test_reduction_pct": 100 * (
|
||||||
|
1 - b["candidate_tests"] / a["candidate_tests"])
|
||||||
|
if a["candidate_tests"] else 0.0,
|
||||||
|
"baseline_tests_per_query": a["candidate_tests"] / a["queries"]
|
||||||
|
if a["queries"] else 0.0,
|
||||||
|
"tests_per_query_after": b["candidate_tests"] / b["queries"]
|
||||||
|
if b["queries"] else 0.0,
|
||||||
|
})
|
||||||
|
output = {"query_and_hit_results_identical": True, "summary": summary}
|
||||||
|
(OUT / "summary.json").write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
|
||||||
|
print("[L104ProbeStatsSummary] " + json.dumps(output, sort_keys=True), flush=True)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Attribute FI/TN L77 sample PCs to current-source functions and work areas."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/census"
|
||||||
|
BINARY = ROOT / "Saved/Linux/l104/profile-bin/VoxelForgeCoreHeadless-l104-profile-clang"
|
||||||
|
SAMPLE = re.compile(
|
||||||
|
r"^\[L77Sample\] pc=(0x\w+) lod=(\d+) stage=(\S+) route=(\S+) "
|
||||||
|
r"phase=(\S+) halo=(\d+) weight=(\d+)$", re.M)
|
||||||
|
TARGETS = {
|
||||||
|
"sweep_base_prepare": "prepareSweptTubeBase",
|
||||||
|
"sweep_lookup": "sweptSample",
|
||||||
|
"exact_route_visit": "VisitNearestPassage",
|
||||||
|
"miss_reach_bound": "PassagePotentialWithinNode",
|
||||||
|
"far_shell_down_hit": "HasDownwardHit",
|
||||||
|
"far_shell_up_hit": "HasUpwardHit",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def load_samples() -> tuple[list[dict], set[str]]:
|
||||||
|
rows, pcs = [], set()
|
||||||
|
for stratum in ("FI", "TN"):
|
||||||
|
for replicate in (1, 2, 3):
|
||||||
|
log = OUT / f"{stratum.lower()}-r{replicate}.log"
|
||||||
|
content = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
for pc, lod, stage, route, phase, halo, weight in SAMPLE.findall(content):
|
||||||
|
if int(lod) > 2:
|
||||||
|
continue
|
||||||
|
pcs.add(pc)
|
||||||
|
rows.append({"stratum": stratum, "replicate": replicate, "pc": pc,
|
||||||
|
"lod": int(lod), "stage": stage, "route": route,
|
||||||
|
"phase": phase, "weight": int(weight)})
|
||||||
|
return rows, pcs
|
||||||
|
|
||||||
|
|
||||||
|
def symbolize(pcs: set[str]) -> dict[str, list[dict]]:
|
||||||
|
command = ["llvm-symbolizer-19", "--demangle", "--inlines", "--output-style=JSON",
|
||||||
|
f"--obj={BINARY}"]
|
||||||
|
result = subprocess.run(command, input="\n".join(sorted(pcs)) + "\n",
|
||||||
|
capture_output=True, text=True, check=True)
|
||||||
|
decoder = json.JSONDecoder()
|
||||||
|
offset = 0
|
||||||
|
output = {}
|
||||||
|
while offset < len(result.stdout):
|
||||||
|
while offset < len(result.stdout) and result.stdout[offset].isspace():
|
||||||
|
offset += 1
|
||||||
|
if offset == len(result.stdout):
|
||||||
|
break
|
||||||
|
item, offset = decoder.raw_decode(result.stdout, offset)
|
||||||
|
output[item["Address"]] = item.get("Symbol", [])
|
||||||
|
if output.keys() != pcs:
|
||||||
|
raise RuntimeError(f"symbolizer resolved {len(output)}/{len(pcs)} sampled PCs")
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
def write_csv(path: Path, rows: list[dict]) -> None:
|
||||||
|
with path.open("w", encoding="utf-8", newline="") as stream:
|
||||||
|
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not BINARY.is_file():
|
||||||
|
raise SystemExit(f"missing profile binary: {BINARY}")
|
||||||
|
samples, pcs = load_samples()
|
||||||
|
symbols = symbolize(pcs)
|
||||||
|
function_totals = Counter()
|
||||||
|
target_totals = Counter()
|
||||||
|
for sample in samples:
|
||||||
|
frames = symbols[sample["pc"]]
|
||||||
|
top_function = frames[0].get("FunctionName", "?") if frames else "?"
|
||||||
|
function_totals[(sample["stratum"], sample["stage"], sample["phase"],
|
||||||
|
top_function)] += sample["weight"]
|
||||||
|
seen = set()
|
||||||
|
for name, token in TARGETS.items():
|
||||||
|
matches = [frame for frame in frames if token in frame.get("FunctionName", "")]
|
||||||
|
if not matches:
|
||||||
|
continue
|
||||||
|
frame = matches[0]
|
||||||
|
line = int(frame.get("Line", 0))
|
||||||
|
category = name
|
||||||
|
if name == "sweep_base_prepare":
|
||||||
|
category = ("sweep_prepare_base" if 14249 <= line <= 14435 else
|
||||||
|
"sweep_prepare_detail" if 14436 <= line <= 14521 else
|
||||||
|
"sweep_prepare_unlocated")
|
||||||
|
key = (sample["stratum"], sample["stage"], sample["phase"], category)
|
||||||
|
if key not in seen:
|
||||||
|
target_totals[key] += sample["weight"]
|
||||||
|
seen.add(key)
|
||||||
|
|
||||||
|
by_function = [
|
||||||
|
{"stratum": stratum, "stage": stage, "phase": phase,
|
||||||
|
"weighted_ticks": ticks, "top_function": function}
|
||||||
|
for (stratum, stage, phase, function), ticks in
|
||||||
|
sorted(function_totals.items(), key=lambda item: (item[0][:3], -item[1]))
|
||||||
|
]
|
||||||
|
by_target = [
|
||||||
|
{"stratum": stratum, "stage": stage, "phase": phase,
|
||||||
|
"work_area": target, "weighted_ticks": ticks}
|
||||||
|
for (stratum, stage, phase, target), ticks in sorted(target_totals.items())
|
||||||
|
]
|
||||||
|
write_csv(OUT / "symbol-function-totals.csv", by_function)
|
||||||
|
write_csv(OUT / "symbol-work-area-totals.csv", by_target)
|
||||||
|
print(f"[L103SymbolAnalysis] samples={len(samples)} unique_pcs={len(pcs)} "
|
||||||
|
f"functions={len(by_function)} work_areas={len(by_target)}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Split worker idle time across compile, near tile, far-parent and LOD2 phases."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/timeline"
|
||||||
|
EVENT = re.compile(
|
||||||
|
r"^\[CpuTileTimeline\] event=(\S+) key=\(([^,]+),([^,]+),([^,]+),lod(-?\d+)\) "
|
||||||
|
r"at_ms=([0-9.]+).*?worker_id=(-?\d+) queue_depth=(\d+) active_workers=(\d+) "
|
||||||
|
r"priority_rank=(\d+)", re.M)
|
||||||
|
COLD = re.compile(
|
||||||
|
r"^\[ColdTimeline\] event=(\S+) edge=(\S+) at_ms=([0-9.]+)", re.M)
|
||||||
|
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||||
|
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||||
|
|
||||||
|
|
||||||
|
def run(path: Path) -> dict:
|
||||||
|
content = path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
events = []
|
||||||
|
for match in EVENT.finditer(content):
|
||||||
|
event, x, y, z, lod, ms, worker, queue, active, rank = match.groups()
|
||||||
|
events.append({"event": event, "lod": int(lod), "at_ms": float(ms),
|
||||||
|
"worker": int(worker), "queue": int(queue),
|
||||||
|
"active": int(active), "rank": int(rank)})
|
||||||
|
cold = {f"{event}_{edge}": float(ms) for event, edge, ms in COLD.findall(content)}
|
||||||
|
idle_start: dict[int, float] = {}
|
||||||
|
idle_intervals = []
|
||||||
|
for event in sorted((e for e in events if e["event"] in ("idle_begin", "idle_end")),
|
||||||
|
key=lambda row: row["at_ms"]):
|
||||||
|
if event["event"] == "idle_begin":
|
||||||
|
idle_start[event["worker"]] = event["at_ms"]
|
||||||
|
elif event["worker"] in idle_start:
|
||||||
|
idle_intervals.append((idle_start.pop(event["worker"]), event["at_ms"]))
|
||||||
|
|
||||||
|
near = NEAR.search(content)
|
||||||
|
pipeline = PIPELINE.search(content)
|
||||||
|
config = re.search(r"^\[HeadlessConfig\] (.*)$", content, re.M)
|
||||||
|
if not events or near is None or pipeline is None or config is None:
|
||||||
|
raise RuntimeError(f"incomplete timeline markers: {path}")
|
||||||
|
config_fields = dict(FIELDS.findall(config.group(1)))
|
||||||
|
workers = int(config_fields["workers"])
|
||||||
|
whole_fields = dict(FIELDS.findall(pipeline.group(1)))
|
||||||
|
requested = [e for e in events if e["event"] == "requested"]
|
||||||
|
admitted = [e for e in events if e["event"] == "admitted"]
|
||||||
|
lod01_admitted = sum(e["lod"] <= 1 for e in admitted)
|
||||||
|
lod01_requested = sum(e["lod"] <= 1 for e in requested)
|
||||||
|
phases = (
|
||||||
|
("full_pipeline", "desired_tile_set_requested", "lod0_2_done_pipeline_drain"),
|
||||||
|
("base_shape_compile", "base_compile_begin", "base_compile_end"),
|
||||||
|
("lod0_1_tile_drain", "lod0_1_admission_end", "lod0_1_done_publish_drained"),
|
||||||
|
("far_parent_build", "far_parent_build_begin", "far_parent_build_end"),
|
||||||
|
("lod2_queue_and_drain", "lod2_queue_begin", "lod2_done_pipeline_drain"),
|
||||||
|
)
|
||||||
|
phase_rows = []
|
||||||
|
for label, begin, end in phases:
|
||||||
|
start, finish = cold[begin], cold[end]
|
||||||
|
idle_worker_ms = sum(max(0.0, min(b, finish) - max(a, start))
|
||||||
|
for a, b in idle_intervals)
|
||||||
|
worker_capacity_ms = max(0.0, finish - start) * workers
|
||||||
|
phase_rows.append({"phase": label, "wall_ms": finish - start,
|
||||||
|
"idle_worker_ms": idle_worker_ms,
|
||||||
|
"idle_pct": 100 * idle_worker_ms / worker_capacity_ms
|
||||||
|
if worker_capacity_ms else 0.0,
|
||||||
|
"busy_pct": 100 * (1 - idle_worker_ms / worker_capacity_ms)
|
||||||
|
if worker_capacity_ms else 0.0})
|
||||||
|
|
||||||
|
# Report the first and last near-tile dispatches and the final queued jobs.
|
||||||
|
lod01_starts = [e for e in events if e["event"] == "started" and e["lod"] <= 1]
|
||||||
|
lod01_finishes = [e for e in events if e["event"] == "finished" and e["lod"] <= 1]
|
||||||
|
near_late = [e for e in lod01_starts if e["at_ms"] >= cold["lod0_1_done_publish_drained"] - 100]
|
||||||
|
return {
|
||||||
|
"log": str(path.relative_to(ROOT)), "workers": workers,
|
||||||
|
"source_stamp": re.search(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", content, re.M).group(1),
|
||||||
|
"requested_lod01_tiles": lod01_requested, "admitted_lod01_tiles": lod01_admitted,
|
||||||
|
"lod01_started": len(lod01_starts), "lod01_finished": len(lod01_finishes),
|
||||||
|
"near_tiles": int(near.group(1)), "whole_tiles": whole_fields["whole_tiles"],
|
||||||
|
"lod01_done_ms": cold["lod0_1_done_publish_drained"],
|
||||||
|
"lod01_last_dispatch_before_done_ms": max(
|
||||||
|
(e["at_ms"] for e in lod01_starts), default=0.0),
|
||||||
|
"lod01_starts_in_last_100ms": len(near_late),
|
||||||
|
"worker_idle_phases": phase_rows,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
rows = [run(path) for path in sorted(OUT.glob("*-w*-direct-measured-r1.log"))]
|
||||||
|
if len(rows) != 4:
|
||||||
|
raise SystemExit(f"expected FI/TN W8/W12 timeline logs, found {len(rows)}")
|
||||||
|
(OUT / "analysis.json").write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8")
|
||||||
|
with (OUT / "utilization.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||||
|
fields = ["log", "workers", "requested_lod01_tiles", "admitted_lod01_tiles",
|
||||||
|
"phase", "wall_ms", "idle_worker_ms", "idle_pct", "busy_pct"]
|
||||||
|
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||||
|
writer.writeheader()
|
||||||
|
for row in rows:
|
||||||
|
for phase in row["worker_idle_phases"]:
|
||||||
|
writer.writerow({"log": row["log"], "workers": row["workers"],
|
||||||
|
"requested_lod01_tiles": row["requested_lod01_tiles"],
|
||||||
|
"admitted_lod01_tiles": row["admitted_lod01_tiles"], **phase})
|
||||||
|
print("[L104TimelineAnalysis] " + json.dumps(rows, sort_keys=True), flush=True)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Compare every L104 tile hash against its L103 Clang/GCC W1/W8/W12 peer."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
BEFORE = ROOT / "Saved/Linux/l103/hashes"
|
||||||
|
AFTER = ROOT / "Saved/Linux/l104/hashes"
|
||||||
|
TILE = re.compile(
|
||||||
|
r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
|
||||||
|
r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) "
|
||||||
|
r"topology=(0x[0-9a-f]+)$", re.M)
|
||||||
|
WHOLE = re.compile(
|
||||||
|
r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) "
|
||||||
|
r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M)
|
||||||
|
|
||||||
|
|
||||||
|
def parse(path: Path) -> tuple[dict, tuple[str, ...]]:
|
||||||
|
content = path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
tiles = {(int(lod), int(x), int(y), int(z)): values
|
||||||
|
for lod, x, y, z, *values in TILE.findall(content)}
|
||||||
|
whole = WHOLE.search(content)
|
||||||
|
if not tiles or whole is None:
|
||||||
|
raise RuntimeError(f"hash data missing in {path}")
|
||||||
|
return tiles, whole.groups()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
runs = json.loads((AFTER / "summary.json").read_text(encoding="utf-8"))["runs"]
|
||||||
|
comparisons = []
|
||||||
|
for run in runs:
|
||||||
|
stem = f"{run['stratum'].lower()}-{run['compiler'].lower()}-w{run['workers']}.log"
|
||||||
|
old_tiles, old_whole = parse(BEFORE / stem)
|
||||||
|
new_tiles, new_whole = parse(ROOT / run["log"])
|
||||||
|
keys = set(old_tiles) | set(new_tiles)
|
||||||
|
tile_mismatches = sum(old_tiles.get(key) != new_tiles.get(key) for key in keys)
|
||||||
|
comparisons.append({
|
||||||
|
"stratum": run["stratum"], "compiler": run["compiler"],
|
||||||
|
"workers": run["workers"], "tiles_before": len(old_tiles),
|
||||||
|
"tiles_after": len(new_tiles), "tile_mismatch_keys": tile_mismatches,
|
||||||
|
"missing_after": len(set(old_tiles) - set(new_tiles)),
|
||||||
|
"extra_after": len(set(new_tiles) - set(old_tiles)),
|
||||||
|
"whole_hash_before": old_whole, "whole_hash_after": new_whole,
|
||||||
|
"whole_hash_match": old_whole == new_whole,
|
||||||
|
})
|
||||||
|
passed = len(comparisons) == 24 and all(
|
||||||
|
row["tiles_before"] == row["tiles_after"] and row["tile_mismatch_keys"] == 0
|
||||||
|
and row["missing_after"] == 0 and row["extra_after"] == 0
|
||||||
|
and row["whole_hash_match"] for row in comparisons)
|
||||||
|
result = {"all_24_l103_peer_comparisons_identical": passed,
|
||||||
|
"comparisons": comparisons}
|
||||||
|
(AFTER / "baseline-comparison.json").write_text(
|
||||||
|
json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||||
|
print("[L104HashBaselineComparison] " + json.dumps(
|
||||||
|
{"pass": passed, "comparisons": len(comparisons),
|
||||||
|
"tile_mismatch_keys": sum(r["tile_mismatch_keys"] for r in comparisons),
|
||||||
|
"whole_hash_mismatches": sum(not r["whole_hash_match"] for r in comparisons)},
|
||||||
|
sort_keys=True), flush=True)
|
||||||
|
return 0 if passed else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
# Linux104 results
|
||||||
|
|
||||||
|
Final source stamp: `b23900c9d2ee10b0ee29bec1b1b7f74ee3fa697f5c66682e808b19ad9c334780`.
|
||||||
|
|
||||||
|
## Far-shell refinement
|
||||||
|
|
||||||
|
Near tiles still need floor and ceiling boundary patches: an owner tile can have an open stratum boundary, and the coverage checks preserve existing faces while filling only gaps. LOD is not a safe skip condition. The waste was candidate search: each `HasDownwardHit` / `HasUpwardHit` query scanned the tile’s full XY column, including triangles at unrelated heights. Exact-key/per-column memoization reused too few LOD0–1 queries, so it was replaced with direction-specific 3D cell bins in [FarShellsInternal.h](../../../Core/src/FarShellsInternal.h). The bins conservatively index triangle bounds; the existing normal, barycentric and tolerance predicates still decide every hit.
|
||||||
|
|
||||||
|
Cache-off and indexed runs had identical query and hit-result counts in all four views. FI made 1,915,890 LOD0–2 hit queries; TN made 2,037,135. Candidate triangle visits per query were:
|
||||||
|
|
||||||
|
| View | LOD | Queries | Before | Indexed | Fewer visits |
|
||||||
|
|---|---:|---:|---:|---:|---:|
|
||||||
|
| FI | 0 | 1,308,990 | 182.56 | 21.87 | 88.0% |
|
||||||
|
| FI | 1 | 484,020 | 179.38 | 5.51 | 96.9% |
|
||||||
|
| FI | 2 | 122,880 | 4.76 | 0.07 | 98.6% |
|
||||||
|
| TN | 0 | 1,260,650 | 203.96 | 19.16 | 90.6% |
|
||||||
|
| TN | 1 | 490,390 | 178.66 | 4.34 | 97.6% |
|
||||||
|
| TN | 2 | 286,095 | 4.91 | 0.04 | 99.1% |
|
||||||
|
|
||||||
|
Weighted symbol samples over three W12 census runs put the hit functions at about **0.37 s FI / 0.34 s TN per run**, versus L103’s 1.73 / 1.85 s. L80 paired readiness also improved FI/TN near-ready time by 6.7–8.3% and whole-view time by 5.5–7.0% at W8/W12.
|
||||||
|
|
||||||
|
## Worker idle
|
||||||
|
|
||||||
|
The reported 24–28% is full-pipeline idle, not idle in the LOD0–1 tile drain. All 960 requested LOD0–1 tiles were admitted, started and finished. The drain kept workers 99.4% busy or better; the last dispatch preceded drain completion by only 8–14 ms. The idle is concentrated in the serial base-shape phase and the later far-parent phase, after or before the near tile drain. Near-first gating therefore does not starve the critical near drain, and changing admission or heavy-tile splits would not fill a materially idle pool there. Scheduling is unchanged.
|
||||||
|
|
||||||
|
| View | Workers | Full-pipeline idle | LOD0–1 drain idle | LOD0–1 tiles |
|
||||||
|
|---|---:|---:|---:|---:|
|
||||||
|
| FI | 8 | 24.2% | 0.38% | 960/960 |
|
||||||
|
| FI | 12 | 26.5% | 0.60% | 960/960 |
|
||||||
|
| TN | 8 | 24.8% | 0.60% | 960/960 |
|
||||||
|
| TN | 12 | 28.3% | 0.57% | 960/960 |
|
||||||
|
|
||||||
|
The timeline’s base-shape phase left workers idle for 232–251 ms; the far-parent phase left them idle for 110–144 ms. See `Saved/Linux/l104/timeline/utilization.csv` and `analysis.json`.
|
||||||
|
|
||||||
|
## Census closure
|
||||||
|
|
||||||
|
Medians; L103 has three W12 runs per stratum. L104 has three per stratum, with six additional FI runs to check closure. All L104 runs had zero lost samples and analyzer validation passed.
|
||||||
|
|
||||||
|
| View | Runs L103 → L104 | Whole near-worker closure L103 → L104 | Grid-fill closure L103 → L104 |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| SW | 3 → 3 | 95.32% → 98.18% | 104.06% → 100.54% |
|
||||||
|
| FI | 3 → 9 | 99.17% → 98.84% | 100.04% → 99.41% |
|
||||||
|
| TN | 3 → 3 | 99.11% → 99.60% | 99.76% → 100.09% |
|
||||||
|
| VS | 3 → 3 | 99.12% → 99.19% | 100.28% → 100.00% |
|
||||||
|
|
||||||
|
FI grid-fill closure clears 99%; its broader whole near-worker sample closure is 98.84%, 0.16 percentage points below 99% after nine runs. This remains the census shortfall.
|
||||||
|
|
||||||
|
## L80 readiness
|
||||||
|
|
||||||
|
Three measured interleaved before/after pairs per cell; medians in seconds. Each run completed the full near and whole tile sets.
|
||||||
|
|
||||||
|
| View | Workers | Near LOD0–2 before → after (change) | Whole view before → after (change) |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| SW | 8 | 0.858 → 0.845 (−1.4%) | 2.046 → 2.012 (−1.7%) |
|
||||||
|
| SW | 12 | 0.844 → 0.845 (+0.1%) | 1.922 → 1.882 (−2.1%) |
|
||||||
|
| FI | 8 | 1.751 → 1.607 (−8.2%) | 2.809 → 2.650 (−5.7%) |
|
||||||
|
| FI | 12 | 1.633 → 1.524 (−6.7%) | 2.603 → 2.461 (−5.5%) |
|
||||||
|
| TN | 8 | 1.669 → 1.538 (−7.9%) | 2.661 → 2.482 (−6.7%) |
|
||||||
|
| TN | 12 | 1.505 → 1.380 (−8.3%) | 2.402 → 2.233 (−7.0%) |
|
||||||
|
| VS | 8 | 1.156 → 1.092 (−5.5%) | 2.164 → 2.069 (−4.4%) |
|
||||||
|
| VS | 12 | 0.995 → 0.920 (−7.6%) | 1.900 → 1.783 (−6.2%) |
|
||||||
|
|
||||||
|
## Hashes
|
||||||
|
|
||||||
|
Every tile and whole-view hash matched the L103 peer for all 24 stratum/compiler/worker combinations (Clang and GCC, W1/W8/W12). The current 20 cross-compiler/worker comparisons also matched. The table gives the shared whole-view hashes; order is density, mesh, payload, topology.
|
||||||
|
|
||||||
|
| View | Tiles | Density | Mesh | Payload | Topology |
|
||||||
|
|---|---:|---|---|---|---|
|
||||||
|
| SW | 2,850 | `0x8723d8c61348bf31` | `0xfe1c3e410373a343` | `0xbf111bbc45e2f017` | `0xc9c562b100b51f7f` |
|
||||||
|
| FI | 2,249 | `0x6bb633f756e2b8be` | `0xfe930e83b0dc6bf3` | `0x5298602285b6892c` | `0xe306f87ae432a4fc` |
|
||||||
|
| TN | 2,233 | `0x612b2d11fc6bd946` | `0xc026a40c6127d649` | `0x0f9036914e28fe46` | `0x94b36e5f73694eff` |
|
||||||
|
| VS | 2,329 | `0x99c93f4c7155ffb3` | `0x488d421c6438f529` | `0x249b5afc12be4540` | `0x9c5e22f6dc1e44b8` |
|
||||||
|
|
||||||
|
## Verification and Unreal
|
||||||
|
|
||||||
|
- Fast tier: **PASS**, 275.5 s (`Saved/Linux/l104/fast-summary.json`). Its deliberate negative controls were rejected as expected.
|
||||||
|
- Linux Unreal build: **Succeeded** with `-NoUBA -MaxParallelActions=4`; UBT compiled `VFCore_Core_src_VoxelForgeCore_cpp.cpp`. The build and three FI arrival traces used CPU affinity only, with no `prlimit`.
|
||||||
|
|
||||||
|
| FI arrival | Near settle from first admission | Full LOD0–2 settled | Near tiles | Exit |
|
||||||
|
|---:|---:|---:|---:|---:|
|
||||||
|
| 1 | 1.572 s | 1.978 s | 1,216 | 0 |
|
||||||
|
| 2 | 1.549 s | 1.964 s | 1,216 | 0 |
|
||||||
|
| 3 | 1.554 s | 1.974 s | 1,216 | 0 |
|
||||||
|
|
||||||
|
## Failed or retired runs
|
||||||
|
|
||||||
|
- The first exact-key/per-column memo passed parity but did not improve readiness; the one-slot variant cut only about 1% of LOD0–1 candidate visits. Both were replaced by the 3D index.
|
||||||
|
- One GCC build was rejected by the source-stamp guard because the source changed during that build. Its binary was discarded; the final GCC build passed on the final stamp.
|
||||||
|
- The first Unreal build stopped before compilation because UBT could not rotate its log in read-only `~/.config/Epic`. Retrying with `-Log` under `Saved/Linux/l104/logs` succeeded.
|
||||||
|
- The final FI whole near-worker sample closure remains below the requested 99% threshold as shown above; the added six runs did not lift its median to 99%.
|
||||||
|
|
||||||
|
All run logs and machine-readable summaries are under `Saved/Linux/l104/`.
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run current-source W12 L77 route/phase and exact-call censuses."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/census"
|
||||||
|
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||||
|
BINARY = ROOT / "Saved/Linux/l104/profile-bin/VoxelForgeCoreHeadless-l104-profile-clang"
|
||||||
|
CAMERAS = {
|
||||||
|
"SW": (8.0, 16.0, -372.2626953125),
|
||||||
|
"FI": (25.8369140625, 3.19921875, -737.0),
|
||||||
|
"TN": (19.166015625, -14.2001953125, -753.3740234375),
|
||||||
|
"VS": (-20.0625, -5.068359375, -900.353515625),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str, replicate: int) -> dict:
|
||||||
|
OUT.mkdir(parents=True, exist_ok=True)
|
||||||
|
log = OUT / f"{stratum.lower()}-r{replicate}.log"
|
||||||
|
camera = ",".join(f"{value:.12f}" for value in CAMERAS[stratum]) + ",0,0"
|
||||||
|
command = [
|
||||||
|
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s",
|
||||||
|
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||||
|
"--stream-pipeline", "--workers=12", "--clip-radius=3", "--max-clip-level=8",
|
||||||
|
"--max-level=8", f"--camera={camera}",
|
||||||
|
"--profile-lod-work", "--profile-passage-composer", "--linux77-profile",
|
||||||
|
]
|
||||||
|
started = time.monotonic()
|
||||||
|
with log.open("w", encoding="utf-8") as stream:
|
||||||
|
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
|
||||||
|
check=False, timeout=1740)
|
||||||
|
row = {"stratum": stratum, "replicate": replicate, "workers": 12,
|
||||||
|
"exit_code": result.returncode,
|
||||||
|
"wall_s": round(time.monotonic() - started, 3),
|
||||||
|
"log": str(log.relative_to(ROOT))}
|
||||||
|
print("[L103CensusRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||||
|
if result.returncode:
|
||||||
|
raise RuntimeError(f"census run failed: {log}, exit={result.returncode}")
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not BINARY.is_file() or not DATA.is_file():
|
||||||
|
raise SystemExit(f"missing input: {BINARY} or {DATA}")
|
||||||
|
runs = []
|
||||||
|
# Three balanced rounds cover every stratum; reverse order in the middle.
|
||||||
|
orders = (("SW", "FI", "TN", "VS"), ("VS", "TN", "FI", "SW"),
|
||||||
|
("FI", "SW", "VS", "TN"))
|
||||||
|
for replicate, order in enumerate(orders, 1):
|
||||||
|
for stratum in order:
|
||||||
|
runs.append(run(stratum, replicate))
|
||||||
|
(OUT / "runs-partial.json").write_text(
|
||||||
|
json.dumps(runs, indent=2) + "\n", encoding="utf-8")
|
||||||
|
(OUT / "runs.json").write_text(json.dumps(runs, indent=2) + "\n", encoding="utf-8")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Add FI W12 L77 samples to tighten the final-source closure estimate."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from RunL104Census import run
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/census"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
rows = [run("FI", replicate) for replicate in range(4, 10)]
|
||||||
|
(OUT / "runs-extra-fi.json").write_text(json.dumps(rows, indent=2) + "\n",
|
||||||
|
encoding="utf-8")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Check whole-view tile hashes across current Clang/GCC W1/W8/W12 binaries."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/hashes"
|
||||||
|
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||||
|
BINARIES = {
|
||||||
|
"Clang": ROOT / "Saved/Linux/l104/bin/VoxelForgeCoreHeadless-l104-clang",
|
||||||
|
"GCC": ROOT / "Saved/Linux/l104/bin/VoxelForgeCoreHeadless-l104-gcc",
|
||||||
|
}
|
||||||
|
CAMERAS = {
|
||||||
|
"SW": (8.0, 16.0, -372.2626953125),
|
||||||
|
"FI": (25.8369140625, 3.19921875, -737.0),
|
||||||
|
"TN": (19.166015625, -14.2001953125, -753.3740234375),
|
||||||
|
"VS": (-20.0625, -5.068359375, -900.353515625),
|
||||||
|
}
|
||||||
|
TILE_HASH = re.compile(
|
||||||
|
r"^\[PipelineTileHash\] lod=(-?\d+) x=(-?\d+) y=(-?\d+) z=(-?\d+) "
|
||||||
|
r"density=(0x[0-9a-f]+) mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) "
|
||||||
|
r"topology=(0x[0-9a-f]+)$", re.M)
|
||||||
|
WHOLE_HASH = re.compile(
|
||||||
|
r"^\[PipelineWholeViewHash\] tiles=(\d+) density=(0x[0-9a-f]+) "
|
||||||
|
r"mesh=(0x[0-9a-f]+) payload=(0x[0-9a-f]+) topology=(0x[0-9a-f]+)$", re.M)
|
||||||
|
FIELDS = ("density", "mesh", "payload", "topology")
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str, compiler: str, workers: int) -> dict:
|
||||||
|
OUT.mkdir(parents=True, exist_ok=True)
|
||||||
|
log = OUT / f"{stratum.lower()}-{compiler.lower()}-w{workers}.log"
|
||||||
|
camera = ",".join(f"{value:.12f}" for value in CAMERAS[stratum]) + ",0,0"
|
||||||
|
command = [
|
||||||
|
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||||
|
"timeout", "1740s", str(BINARIES[compiler]), str(DATA),
|
||||||
|
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
|
||||||
|
f"--workers={workers}", "--clip-radius=3", "--max-clip-level=8",
|
||||||
|
"--max-level=8", f"--camera={camera}", "--stream-hash-tiles",
|
||||||
|
]
|
||||||
|
started = time.monotonic()
|
||||||
|
with log.open("w", encoding="utf-8") as stream:
|
||||||
|
result = subprocess.run(command, cwd=ROOT, stdout=stream,
|
||||||
|
stderr=subprocess.STDOUT, check=False, timeout=1740)
|
||||||
|
output = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
hashes = {
|
||||||
|
(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))):
|
||||||
|
m.group(5, 6, 7, 8) for m in TILE_HASH.finditer(output)
|
||||||
|
}
|
||||||
|
whole = WHOLE_HASH.search(output)
|
||||||
|
stamp = re.search(r"^\[VoxelForgeSourceStamp\] sha256=([0-9a-f]{64})$", output, re.M)
|
||||||
|
if (result.returncode or whole is None or stamp is None
|
||||||
|
or len(hashes) != int(whole.group(1))):
|
||||||
|
raise RuntimeError(f"hash run failed or incomplete: exit={result.returncode} {log}")
|
||||||
|
row = {
|
||||||
|
"stratum": stratum, "compiler": compiler, "workers": workers,
|
||||||
|
"source_stamp": stamp.group(1), "exit_code": result.returncode,
|
||||||
|
"wall_s": round(time.monotonic() - started, 3), "tile_count": len(hashes),
|
||||||
|
"whole_hash": list(whole.groups()),
|
||||||
|
"tile_hashes": {"|".join(map(str, key)): value for key, value in hashes.items()},
|
||||||
|
"log": str(log.relative_to(ROOT)),
|
||||||
|
}
|
||||||
|
print("[L103HashRun] " + json.dumps(
|
||||||
|
{key: value for key, value in row.items() if key != "tile_hashes"},
|
||||||
|
sort_keys=True), flush=True)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def compare(stratum: str, reference: dict, candidate: dict) -> dict:
|
||||||
|
ref = {tuple(map(int, key.split("|"))): value
|
||||||
|
for key, value in reference["tile_hashes"].items()}
|
||||||
|
cand = {tuple(map(int, key.split("|"))): value
|
||||||
|
for key, value in candidate["tile_hashes"].items()}
|
||||||
|
keys = set(ref) | set(cand)
|
||||||
|
mismatches = {
|
||||||
|
field: sum(ref.get(key, (None,) * 4)[index] != cand.get(key, (None,) * 4)[index]
|
||||||
|
for key in keys)
|
||||||
|
for index, field in enumerate(FIELDS)
|
||||||
|
}
|
||||||
|
row = {
|
||||||
|
"stratum": stratum,
|
||||||
|
"reference": f"{reference['compiler']}-w{reference['workers']}",
|
||||||
|
"candidate": f"{candidate['compiler']}-w{candidate['workers']}",
|
||||||
|
"tile_counts_match": reference["tile_count"] == candidate["tile_count"],
|
||||||
|
"whole_hashes_match": reference["whole_hash"] == candidate["whole_hash"],
|
||||||
|
"tile_hashes_match": all(value == 0 for value in mismatches.values()) and ref.keys() == cand.keys(),
|
||||||
|
"component_mismatch_tiles": mismatches,
|
||||||
|
"tiles_reference": reference["tile_count"], "tiles_candidate": candidate["tile_count"],
|
||||||
|
}
|
||||||
|
print("[L103HashComparison] " + json.dumps(row, sort_keys=True), flush=True)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
for path in (*BINARIES.values(), DATA):
|
||||||
|
if not path.is_file():
|
||||||
|
raise SystemExit(f"missing input: {path}")
|
||||||
|
details, comparisons = [], []
|
||||||
|
for stratum in CAMERAS:
|
||||||
|
group = {(compiler, workers): run(stratum, compiler, workers)
|
||||||
|
for compiler in BINARIES for workers in (1, 8, 12)}
|
||||||
|
reference = group[("Clang", 1)]
|
||||||
|
for key, candidate in group.items():
|
||||||
|
if key != ("Clang", 1):
|
||||||
|
comparisons.append(compare(stratum, reference, candidate))
|
||||||
|
details.extend({key: value for key, value in row.items() if key != "tile_hashes"}
|
||||||
|
for row in group.values())
|
||||||
|
(OUT / "runs-partial.json").write_text(json.dumps(
|
||||||
|
{"runs": details, "comparisons": comparisons}, indent=2) + "\n", encoding="utf-8")
|
||||||
|
summary = {"runs": details, "comparisons": comparisons}
|
||||||
|
(OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||||
|
return 0 if all(row["tile_counts_match"] and row["whole_hashes_match"]
|
||||||
|
and row["tile_hashes_match"] for row in comparisons) else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Measure exact far-hit triangle probes with the L104 memo disabled/enabled."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/probes"
|
||||||
|
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||||
|
BINARY = ROOT / "Saved/Linux/l104/bin/VoxelForgeCoreHeadless-l104-clang"
|
||||||
|
CAMERAS = {
|
||||||
|
"SW": (8.0, 16.0, -372.2626953125),
|
||||||
|
"FI": (25.8369140625, 3.19921875, -737.0),
|
||||||
|
"TN": (19.166015625, -14.2001953125, -753.3740234375),
|
||||||
|
"VS": (-20.0625, -5.068359375, -900.353515625),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str, cache: str) -> dict:
|
||||||
|
OUT.mkdir(parents=True, exist_ok=True)
|
||||||
|
log = OUT / f"{stratum.lower()}-{cache}.log"
|
||||||
|
camera = ",".join(f"{value:.12f}" for value in CAMERAS[stratum]) + ",0,0"
|
||||||
|
command = [
|
||||||
|
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--",
|
||||||
|
"env", f"VOXELFORGE_L104_FAR_HIT_CACHE={0 if cache == 'off' else 1}",
|
||||||
|
"VOXELFORGE_L104_FAR_HIT_STATS=1", "timeout", "1740s", str(BINARY), str(DATA),
|
||||||
|
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
|
||||||
|
"--workers=12", "--clip-radius=3", "--max-clip-level=8", "--max-level=8",
|
||||||
|
f"--camera={camera}", "--stream-scheduler-policy=3",
|
||||||
|
]
|
||||||
|
started = time.monotonic()
|
||||||
|
with log.open("w", encoding="utf-8") as stream:
|
||||||
|
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
|
||||||
|
check=False, timeout=1740)
|
||||||
|
content = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
rows = []
|
||||||
|
pattern = re.compile(
|
||||||
|
r"^\[FarHitQueryStats\] lod=(\d+) origin=\(([^)]*)\) "
|
||||||
|
r"down_queries=(\d+) down_candidate_tests=(\d+) down_hit_queries=(\d+) "
|
||||||
|
r"up_queries=(\d+) up_candidate_tests=(\d+) up_hit_queries=(\d+)$", re.M)
|
||||||
|
for match in pattern.finditer(content):
|
||||||
|
rows.append({
|
||||||
|
"lod": int(match.group(1)), "origin": match.group(2),
|
||||||
|
"down_queries": int(match.group(3)), "down_candidate_tests": int(match.group(4)),
|
||||||
|
"down_hit_queries": int(match.group(5)), "up_queries": int(match.group(6)),
|
||||||
|
"up_candidate_tests": int(match.group(7)), "up_hit_queries": int(match.group(8)),
|
||||||
|
})
|
||||||
|
if result.returncode or not rows:
|
||||||
|
raise RuntimeError(f"probe stats failed or missing: {log}; exit={result.returncode}")
|
||||||
|
row = {"stratum": stratum, "cache": cache, "exit_code": result.returncode,
|
||||||
|
"source_stamp": re.search(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", content, re.M).group(1),
|
||||||
|
"wall_s": round(time.monotonic() - started, 3), "indices": len(rows),
|
||||||
|
"stats": rows, "log": str(log.relative_to(ROOT))}
|
||||||
|
(OUT / f"{stratum.lower()}-{cache}.json").write_text(
|
||||||
|
json.dumps(row, indent=2) + "\n", encoding="utf-8")
|
||||||
|
print("[L104ProbeStatsRun] " + json.dumps(
|
||||||
|
{key: value for key, value in row.items() if key != "stats"}, sort_keys=True), flush=True)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not BINARY.is_file() or not DATA.is_file():
|
||||||
|
raise SystemExit(f"missing input: {BINARY} or {DATA}")
|
||||||
|
rows = []
|
||||||
|
for stratum in CAMERAS:
|
||||||
|
for cache in ("off", "on"):
|
||||||
|
rows.append(run(stratum, cache))
|
||||||
|
(OUT / "runs-partial.json").write_text(
|
||||||
|
json.dumps(rows, indent=2) + "\n", encoding="utf-8")
|
||||||
|
(OUT / "runs.json").write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""L80-protocol near and whole-view readiness at W8/W12 for all owner strata."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import statistics
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/readiness"
|
||||||
|
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||||
|
BINARY = ROOT / "Saved/Linux/l104/bin/VoxelForgeCoreHeadless-l104-clang"
|
||||||
|
STRATA = {
|
||||||
|
"SW": {"camera": (8.0, 16.0, -372.2626953125), "near": 1408, "whole": 2850},
|
||||||
|
"FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249},
|
||||||
|
"TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233},
|
||||||
|
"VS": {"camera": (-20.0625, -5.068359375, -900.353515625), "near": 1344, "whole": 2329},
|
||||||
|
}
|
||||||
|
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
PUBLISHED = re.compile(r"^\[WorldAssemblerNearLODPublished\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
PIPELINE_READY = re.compile(r"^\[WorldAssemblerNearLODPipelineReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||||
|
CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M)
|
||||||
|
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||||
|
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str, workers: int, phase: str, replicate: int) -> dict:
|
||||||
|
OUT.mkdir(parents=True, exist_ok=True)
|
||||||
|
suffix = f"-r{replicate}" if phase == "measured" else ""
|
||||||
|
log = OUT / f"{stratum.lower()}-w{workers}-{phase}{suffix}.log"
|
||||||
|
camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0"
|
||||||
|
command = [
|
||||||
|
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s",
|
||||||
|
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||||
|
"--stream-pipeline", f"--workers={workers}", "--clip-radius=3",
|
||||||
|
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
|
||||||
|
"--profile-lod-work", "--stream-scheduler-policy=3",
|
||||||
|
]
|
||||||
|
started = time.monotonic()
|
||||||
|
with log.open("w", encoding="utf-8") as stream:
|
||||||
|
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
|
||||||
|
check=False, timeout=1740)
|
||||||
|
content = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
near, published, pipeline_ready = (NEAR.search(content), PUBLISHED.search(content),
|
||||||
|
PIPELINE_READY.search(content))
|
||||||
|
pipeline, config, stamp = PIPELINE.search(content), CONFIG.search(content), STAMP.search(content)
|
||||||
|
if result.returncode or not all((near, published, pipeline_ready, pipeline, config, stamp)):
|
||||||
|
raise RuntimeError(f"readiness run failed or markers missing: {log}; exit={result.returncode}")
|
||||||
|
fields = dict(FIELDS.findall(pipeline.group(1)))
|
||||||
|
config_fields = dict(FIELDS.findall(config.group(1)))
|
||||||
|
spec = STRATA[stratum]
|
||||||
|
if int(config_fields["workers"]) != workers:
|
||||||
|
raise RuntimeError(f"worker mismatch: {log}")
|
||||||
|
if stamp.group(1) != "b23900c9d2ee10b0ee29bec1b1b7f74ee3fa697f5c66682e808b19ad9c334780":
|
||||||
|
raise RuntimeError(f"source stamp mismatch: {log}: {stamp.group(1)}")
|
||||||
|
for marker in (near, published, pipeline_ready):
|
||||||
|
if (int(marker.group(1)), int(marker.group(2))) != (spec["near"], spec["near"]):
|
||||||
|
raise RuntimeError(f"incomplete near set: {log}: {marker.group(1)}/{marker.group(2)}")
|
||||||
|
whole_done, whole_total = (int(value) for value in fields["whole_tiles"].split("/"))
|
||||||
|
if (whole_done, whole_total) != (spec["whole"], spec["whole"]):
|
||||||
|
raise RuntimeError(f"incomplete whole set: {log}: {whole_done}/{whole_total}")
|
||||||
|
row = {
|
||||||
|
"stratum": stratum, "workers": workers, "phase": phase, "replicate": replicate,
|
||||||
|
"exit_code": result.returncode, "source_stamp": stamp.group(1),
|
||||||
|
"near_tiles": spec["near"], "whole_tiles": spec["whole"],
|
||||||
|
"lod02_ready_s": float(near.group(3)),
|
||||||
|
"lod02_published_s": float(published.group(3)),
|
||||||
|
"lod02_pipeline_ready_s": float(pipeline_ready.group(3)),
|
||||||
|
"whole_view_ready_s": float(fields["whole_view_ready"].removesuffix("s")),
|
||||||
|
"shape_s": float(fields["shape"].removesuffix("s")),
|
||||||
|
"far_shape_s": float(fields["far_shape"].removesuffix("s")),
|
||||||
|
"wall_s": round(time.monotonic() - started, 3),
|
||||||
|
"log": str(log.relative_to(ROOT)),
|
||||||
|
}
|
||||||
|
print("[L104ReadinessRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not DATA.is_file() or not BINARY.is_file():
|
||||||
|
raise SystemExit(f"missing input: {DATA} or {BINARY}")
|
||||||
|
for stratum in ("VS", "TN", "FI", "SW"):
|
||||||
|
for workers in (12, 8):
|
||||||
|
run(stratum, workers, "warmup", 0)
|
||||||
|
schedules = (
|
||||||
|
(("SW", 8), ("SW", 12), ("FI", 8), ("FI", 12), ("TN", 8), ("TN", 12), ("VS", 8), ("VS", 12)),
|
||||||
|
(("VS", 12), ("VS", 8), ("TN", 12), ("TN", 8), ("FI", 12), ("FI", 8), ("SW", 12), ("SW", 8)),
|
||||||
|
(("FI", 8), ("FI", 12), ("SW", 8), ("SW", 12), ("VS", 8), ("VS", 12), ("TN", 8), ("TN", 12)),
|
||||||
|
)
|
||||||
|
rows = []
|
||||||
|
for replicate, schedule in enumerate(schedules, 1):
|
||||||
|
for stratum, workers in schedule:
|
||||||
|
rows.append(run(stratum, workers, "measured", replicate))
|
||||||
|
(OUT / "runs-partial.json").write_text(json.dumps(rows, indent=2) + "\n",
|
||||||
|
encoding="utf-8")
|
||||||
|
summary = []
|
||||||
|
for stratum in STRATA:
|
||||||
|
for workers in (8, 12):
|
||||||
|
group = [row for row in rows if row["stratum"] == stratum and row["workers"] == workers]
|
||||||
|
if len(group) != 3:
|
||||||
|
raise RuntimeError(f"expected three measured runs: {stratum} W{workers}")
|
||||||
|
summary.append({
|
||||||
|
"stratum": stratum, "workers": workers, "measured_runs": 3,
|
||||||
|
**{metric: [row[metric] for row in group]
|
||||||
|
for metric in ("lod02_ready_s", "lod02_published_s",
|
||||||
|
"lod02_pipeline_ready_s", "whole_view_ready_s")},
|
||||||
|
**{f"{metric}_median": statistics.median(row[metric] for row in group)
|
||||||
|
for metric in ("lod02_ready_s", "lod02_published_s",
|
||||||
|
"lod02_pipeline_ready_s", "whole_view_ready_s")},
|
||||||
|
})
|
||||||
|
(OUT / "summary.json").write_text(json.dumps({"runs": rows, "summary": summary}, indent=2)
|
||||||
|
+ "\n", encoding="utf-8")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Interleaved L103/L104 near and whole-view readiness at W8/W12."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import statistics
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/readiness-paired"
|
||||||
|
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||||
|
BINARIES = {
|
||||||
|
"before": ROOT / "Saved/Linux/l101/bin/VoxelForgeCoreHeadless-l101-clang",
|
||||||
|
"after": ROOT / "Saved/Linux/l104/bin/VoxelForgeCoreHeadless-l104-clang",
|
||||||
|
}
|
||||||
|
STAMPS = {
|
||||||
|
"before": "c5f49a45f51856b857a1ee8877b7d777208e67daf64281caefc7ef84e8006410",
|
||||||
|
"after": "b23900c9d2ee10b0ee29bec1b1b7f74ee3fa697f5c66682e808b19ad9c334780",
|
||||||
|
}
|
||||||
|
STRATA = {
|
||||||
|
"SW": {"camera": (8.0, 16.0, -372.2626953125), "near": 1408, "whole": 2850},
|
||||||
|
"FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249},
|
||||||
|
"TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233},
|
||||||
|
"VS": {"camera": (-20.0625, -5.068359375, -900.353515625), "near": 1344, "whole": 2329},
|
||||||
|
}
|
||||||
|
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
PUBLISHED = re.compile(r"^\[WorldAssemblerNearLODPublished\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
PIPELINE_READY = re.compile(r"^\[WorldAssemblerNearLODPipelineReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||||
|
CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M)
|
||||||
|
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||||
|
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str, workers: int, build: str, phase: str, replicate: int) -> dict:
|
||||||
|
OUT.mkdir(parents=True, exist_ok=True)
|
||||||
|
suffix = f"-r{replicate}" if phase == "measured" else ""
|
||||||
|
log = OUT / f"{stratum.lower()}-w{workers}-{build}-{phase}{suffix}.log"
|
||||||
|
camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0"
|
||||||
|
command = [
|
||||||
|
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "env",
|
||||||
|
f"VOXELFORGE_L104_FAR_HIT_CACHE={0 if build == 'before' else 1}",
|
||||||
|
"timeout", "1740s", str(BINARIES[build]), str(DATA),
|
||||||
|
"--world-assembler-prototype", "--stream-benchmark", "--stream-pipeline",
|
||||||
|
f"--workers={workers}", "--clip-radius=3", "--max-clip-level=8", "--max-level=8",
|
||||||
|
f"--camera={camera}", "--profile-lod-work", "--stream-scheduler-policy=3",
|
||||||
|
]
|
||||||
|
started = time.monotonic()
|
||||||
|
with log.open("w", encoding="utf-8") as stream:
|
||||||
|
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
|
||||||
|
check=False, timeout=1740)
|
||||||
|
content = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
near, published, ready = NEAR.search(content), PUBLISHED.search(content), PIPELINE_READY.search(content)
|
||||||
|
pipeline, config, stamp = PIPELINE.search(content), CONFIG.search(content), STAMP.search(content)
|
||||||
|
if result.returncode or not all((near, published, ready, pipeline, config, stamp)):
|
||||||
|
raise RuntimeError(f"readiness run failed or markers missing: {log}; exit={result.returncode}")
|
||||||
|
if stamp.group(1) != STAMPS[build]:
|
||||||
|
raise RuntimeError(f"source stamp mismatch: {log}: {stamp.group(1)}")
|
||||||
|
spec = STRATA[stratum]
|
||||||
|
config_fields = dict(FIELDS.findall(config.group(1)))
|
||||||
|
if int(config_fields["workers"]) != workers:
|
||||||
|
raise RuntimeError(f"worker mismatch: {log}")
|
||||||
|
for marker in (near, published, ready):
|
||||||
|
if (int(marker.group(1)), int(marker.group(2))) != (spec["near"], spec["near"]):
|
||||||
|
raise RuntimeError(f"incomplete near set: {log}: {marker.group(1)}/{marker.group(2)}")
|
||||||
|
pipeline_fields = dict(FIELDS.findall(pipeline.group(1)))
|
||||||
|
whole_done, whole_total = (int(value) for value in pipeline_fields["whole_tiles"].split("/"))
|
||||||
|
if (whole_done, whole_total) != (spec["whole"], spec["whole"]):
|
||||||
|
raise RuntimeError(f"incomplete whole set: {log}: {whole_done}/{whole_total}")
|
||||||
|
row = {
|
||||||
|
"stratum": stratum, "workers": workers, "build": build, "phase": phase,
|
||||||
|
"replicate": replicate, "exit_code": result.returncode,
|
||||||
|
"source_stamp": stamp.group(1), "near_tiles": spec["near"], "whole_tiles": whole_done,
|
||||||
|
"lod02_ready_s": float(near.group(3)),
|
||||||
|
"lod02_published_s": float(published.group(3)),
|
||||||
|
"lod02_pipeline_ready_s": float(ready.group(3)),
|
||||||
|
"whole_view_ready_s": float(pipeline_fields["whole_view_ready"].removesuffix("s")),
|
||||||
|
"shape_s": float(pipeline_fields["shape"].removesuffix("s")),
|
||||||
|
"far_shape_s": float(pipeline_fields["far_shape"].removesuffix("s")),
|
||||||
|
"wall_s": round(time.monotonic() - started, 3),
|
||||||
|
"log": str(log.relative_to(ROOT)),
|
||||||
|
}
|
||||||
|
print("[L104ReadinessRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
for binary in BINARIES.values():
|
||||||
|
if not binary.is_file():
|
||||||
|
raise SystemExit(f"missing binary: {binary}")
|
||||||
|
if not DATA.is_file():
|
||||||
|
raise SystemExit(f"missing owner input: {DATA}")
|
||||||
|
cells = [(stratum, workers) for stratum in ("VS", "TN", "FI", "SW")
|
||||||
|
for workers in (12, 8)]
|
||||||
|
for stratum, workers in cells:
|
||||||
|
for build in ("before", "after"):
|
||||||
|
run(stratum, workers, build, "warmup", 0)
|
||||||
|
orders = (("before", "after"), ("after", "before"), ("before", "after"))
|
||||||
|
rows = []
|
||||||
|
for stratum, workers in cells:
|
||||||
|
for replicate, order in enumerate(orders, 1):
|
||||||
|
for build in order:
|
||||||
|
rows.append(run(stratum, workers, build, "measured", replicate))
|
||||||
|
(OUT / "runs-partial.json").write_text(json.dumps(rows, indent=2) + "\n",
|
||||||
|
encoding="utf-8")
|
||||||
|
summary = []
|
||||||
|
for stratum in STRATA:
|
||||||
|
for workers in (8, 12):
|
||||||
|
group = [row for row in rows if row["stratum"] == stratum and row["workers"] == workers]
|
||||||
|
item = {"stratum": stratum, "workers": workers, "measured_runs_per_build": 3}
|
||||||
|
for build in ("before", "after"):
|
||||||
|
subset = [row for row in group if row["build"] == build]
|
||||||
|
if len(subset) != 3:
|
||||||
|
raise RuntimeError(f"expected 3 measured runs: {stratum} W{workers} {build}")
|
||||||
|
for metric in ("lod02_ready_s", "whole_view_ready_s"):
|
||||||
|
values = [row[metric] for row in subset]
|
||||||
|
item[f"{build}_{metric}"] = values
|
||||||
|
item[f"{build}_{metric}_median"] = statistics.median(values)
|
||||||
|
for metric in ("lod02_ready_s", "whole_view_ready_s"):
|
||||||
|
item[f"{metric}_delta_pct"] = 100 * (
|
||||||
|
item[f"after_{metric}_median"] / item[f"before_{metric}_median"] - 1)
|
||||||
|
summary.append(item)
|
||||||
|
result = {"runs": rows, "summary": summary}
|
||||||
|
(OUT / "summary.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Capture L87 cold critical-path and worker-idle traces on current source."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/timeline"
|
||||||
|
DATA = ROOT / "Standalone/data/owner_world.json"
|
||||||
|
BINARY = ROOT / "Saved/Linux/l104/bin/VoxelForgeCoreHeadless-l104-clang"
|
||||||
|
STRATA = {
|
||||||
|
"FI": {"camera": (25.8369140625, 3.19921875, -737.0), "near": 1216, "whole": 2249},
|
||||||
|
"TN": {"camera": (19.166015625, -14.2001953125, -753.3740234375), "near": 1344, "whole": 2233},
|
||||||
|
}
|
||||||
|
NEAR = re.compile(r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", re.M)
|
||||||
|
PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M)
|
||||||
|
CONFIG = re.compile(r"^\[HeadlessConfig\] (.*)$", re.M)
|
||||||
|
STAMP = re.compile(r"^\[VoxelForgeSourceStamp\] sha256=(\S+)$", re.M)
|
||||||
|
FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)")
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str, workers: int) -> dict:
|
||||||
|
OUT.mkdir(parents=True, exist_ok=True)
|
||||||
|
log = OUT / f"{stratum.lower()}-w{workers}-direct-measured-r1.log"
|
||||||
|
camera = ",".join(f"{value:.12f}" for value in STRATA[stratum]["camera"]) + ",0,0"
|
||||||
|
command = [
|
||||||
|
"taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", "timeout", "1740s",
|
||||||
|
str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark",
|
||||||
|
"--stream-pipeline", f"--workers={workers}", "--clip-radius=3",
|
||||||
|
"--max-clip-level=8", "--max-level=8", f"--camera={camera}",
|
||||||
|
"--profile-lod-work", "--stream-scheduler-policy=3", "--cold-worker-timeline",
|
||||||
|
]
|
||||||
|
started = time.monotonic()
|
||||||
|
with log.open("w", encoding="utf-8") as stream:
|
||||||
|
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT,
|
||||||
|
check=False, timeout=1740)
|
||||||
|
content = log.read_text(encoding="utf-8", errors="replace")
|
||||||
|
near, pipeline, config, stamp = (NEAR.search(content), PIPELINE.search(content),
|
||||||
|
CONFIG.search(content), STAMP.search(content))
|
||||||
|
events = re.findall(r"^\[CpuTileTimeline\] event=(\S+)", content, re.M)
|
||||||
|
cold_events = re.findall(r"^\[ColdTimeline\] event=(\S+) edge=(\S+)", content, re.M)
|
||||||
|
if result.returncode or not all((near, pipeline, config, stamp)) or not events:
|
||||||
|
raise RuntimeError(f"timeline run failed or trace missing: {log}; exit={result.returncode}")
|
||||||
|
if stamp.group(1) != "b23900c9d2ee10b0ee29bec1b1b7f74ee3fa697f5c66682e808b19ad9c334780":
|
||||||
|
raise RuntimeError(f"source stamp mismatch: {log}: {stamp.group(1)}")
|
||||||
|
config_fields = dict(FIELDS.findall(config.group(1)))
|
||||||
|
if int(config_fields["workers"]) != workers:
|
||||||
|
raise RuntimeError(f"worker mismatch: {log}")
|
||||||
|
spec = STRATA[stratum]
|
||||||
|
if (int(near.group(1)), int(near.group(2))) != (spec["near"], spec["near"]):
|
||||||
|
raise RuntimeError(f"incomplete near set: {log}: {near.group(1)}/{near.group(2)}")
|
||||||
|
pipeline_fields = dict(FIELDS.findall(pipeline.group(1)))
|
||||||
|
whole = pipeline_fields["whole_tiles"].split("/")
|
||||||
|
if (int(whole[0]), int(whole[1])) != (spec["whole"], spec["whole"]):
|
||||||
|
raise RuntimeError(f"incomplete whole set: {log}: {pipeline_fields['whole_tiles']}")
|
||||||
|
row = {"stratum": stratum, "workers": workers, "exit_code": result.returncode,
|
||||||
|
"source_stamp": stamp.group(1), "near_tiles": spec["near"],
|
||||||
|
"whole_tiles": spec["whole"], "lod02_ready_s": float(near.group(3)),
|
||||||
|
"whole_view_ready_s": float(pipeline_fields["whole_view_ready"].removesuffix("s")),
|
||||||
|
"timeline_events": len(events), "cold_timeline_marks": len(cold_events),
|
||||||
|
"wall_s": round(time.monotonic() - started, 3),
|
||||||
|
"log": str(log.relative_to(ROOT))}
|
||||||
|
print("[L104TimelineRun] " + json.dumps(row, sort_keys=True), flush=True)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
for stratum in STRATA:
|
||||||
|
for workers in (8, 12):
|
||||||
|
run(stratum, workers)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,484 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run one pinned null-RHI SW/FI arrival for Linux 102."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[4]
|
||||||
|
HOST = Path("/home/jahni/VFHost")
|
||||||
|
HOST_PLUGIN_SAVED = HOST / "Plugins/VoxelForge/Saved/Linux/l104"
|
||||||
|
HOST_SAVED = HOST / "Saved/Linux/l104"
|
||||||
|
EDITOR = Path("/home/jahni/UnrealEngine/Engine/Binaries/Linux/UnrealEditor-Cmd")
|
||||||
|
OUT = ROOT / "Saved/Linux/l104/unreal-arrival"
|
||||||
|
SPAWNS_M = {
|
||||||
|
"FI": (25.8369140625, 3.19921875, -737.0),
|
||||||
|
"SW": (25.8369140625, 3.19921875, -200.0),
|
||||||
|
}
|
||||||
|
MOVE_START_S = 20.0
|
||||||
|
MOVE_DURATION_S = 12.0
|
||||||
|
EXIT_S = MOVE_START_S + MOVE_DURATION_S + 5.0
|
||||||
|
RUN_TIMEOUT_S = 1740
|
||||||
|
|
||||||
|
|
||||||
|
def read_events(path: Path) -> list[dict]:
|
||||||
|
events = []
|
||||||
|
if path.is_file():
|
||||||
|
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||||
|
if line.startswith("{"):
|
||||||
|
try:
|
||||||
|
events.append(json.loads(line))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
def percentile(values: list[float], fraction: float) -> float | None:
|
||||||
|
if not values:
|
||||||
|
return None
|
||||||
|
ordered = sorted(values)
|
||||||
|
return ordered[min(len(ordered) - 1, int(round((len(ordered) - 1) * fraction)))]
|
||||||
|
|
||||||
|
|
||||||
|
def sample_unreal_peak_rss() -> tuple[int, int] | None:
|
||||||
|
executable = str(EDITOR).encode()
|
||||||
|
project = str(HOST / "VFHost.uproject").encode()
|
||||||
|
peak: tuple[int, int] | None = None
|
||||||
|
for process in Path("/proc").glob("[0-9]*"):
|
||||||
|
try:
|
||||||
|
command_line = (process / "cmdline").read_bytes()
|
||||||
|
if executable not in command_line or project not in command_line:
|
||||||
|
continue
|
||||||
|
status = (process / "status").read_text(encoding="ascii", errors="ignore")
|
||||||
|
match = re.search(r"^VmHWM:\s+(\d+)\s+kB$", status, re.MULTILINE)
|
||||||
|
if match:
|
||||||
|
rss_kib = int(match.group(1))
|
||||||
|
if peak is None or rss_kib > peak[0]:
|
||||||
|
peak = (rss_kib, int(process.name))
|
||||||
|
except (FileNotFoundError, PermissionError, ProcessLookupError, OSError):
|
||||||
|
continue
|
||||||
|
return peak
|
||||||
|
|
||||||
|
|
||||||
|
def endphysics_gap(stdout: Path) -> tuple[float | None, int | None]:
|
||||||
|
release_times: dict[int, dict[int, datetime]] = {}
|
||||||
|
pattern = re.compile(
|
||||||
|
r"^\[(\d{4}\.\d{2}\.\d{2}-\d{2}\.\d{2}\.\d{2}:\d{3})\].*"
|
||||||
|
r"LogTick: tick\s+(\d+)\s+-+\s+Release tick group ([34])")
|
||||||
|
with stdout.open(encoding="utf-8", errors="replace") as stream:
|
||||||
|
for line in stream:
|
||||||
|
match = pattern.match(line)
|
||||||
|
if not match:
|
||||||
|
continue
|
||||||
|
timestamp = datetime.strptime(match.group(1), "%Y.%m.%d-%H.%M.%S:%f")
|
||||||
|
frame = int(match.group(2))
|
||||||
|
group = int(match.group(3))
|
||||||
|
release_times.setdefault(frame, {})[group] = timestamp
|
||||||
|
gaps = [(round((times[4] - times[3]).total_seconds() * 1000.0, 3), frame)
|
||||||
|
for frame, times in release_times.items() if 3 in times and 4 in times]
|
||||||
|
return max(gaps, default=(None, None))
|
||||||
|
|
||||||
|
|
||||||
|
def run(stratum: str, repeat: int, phase: str, ready_depth: int,
|
||||||
|
background_cap: int, worker_reserve: int, helper_background: int,
|
||||||
|
tile_background_low: int, tile_background_normal: int, startup_boost: int,
|
||||||
|
tick_logs: bool, insights: bool) -> dict:
|
||||||
|
label = (f"{phase}-{stratum.lower()}-rd{ready_depth}-bg{background_cap}"
|
||||||
|
f"-reserve{worker_reserve}-helperbg{helper_background}"
|
||||||
|
f"-tilelow{tile_background_low}-tilenormal{tile_background_normal}"
|
||||||
|
f"-ticklog{int(tick_logs)}"
|
||||||
|
f"-boost{startup_boost}-{repeat}")
|
||||||
|
trace = OUT / f"{label}.trace.jsonl"
|
||||||
|
utrace = OUT / f"{label}.utrace"
|
||||||
|
abslog = OUT / f"{label}.log"
|
||||||
|
stdout = OUT / f"{label}.stdout.log"
|
||||||
|
output = HOST_PLUGIN_SAVED / "output" / label
|
||||||
|
userdir = HOST_SAVED / "user" / label
|
||||||
|
zendata = HOST_SAVED / "zen" / label
|
||||||
|
ddc = HOST_SAVED / "DDC"
|
||||||
|
for directory in (OUT, output, userdir, zendata, ddc):
|
||||||
|
directory.mkdir(parents=True, exist_ok=True)
|
||||||
|
trace.unlink(missing_ok=True)
|
||||||
|
utrace.unlink(missing_ok=True)
|
||||||
|
spawn_cm = ",".join(f"{value * 100.0:.3f}" for value in SPAWNS_M[stratum])
|
||||||
|
command = [
|
||||||
|
str(EDITOR), str(HOST / "VFHost.uproject"), "-nullrhi", "-unattended",
|
||||||
|
"-nop4", "-nosplash", "-NoSound", "-corelimit=12", "-NoZenAutoLaunch",
|
||||||
|
"-notraceserver", "-nocrashreports",
|
||||||
|
"-DisablePlugins=AndroidFileServer",
|
||||||
|
"-ddc=NoZenLocalFallback", f"-userdir={userdir}", f"-ZenDataPath={zendata}",
|
||||||
|
f"-LocalDataCachePath={ddc}", "-run=VoxelForgeExplore", "-seed=0",
|
||||||
|
"-archetype=TunnelNetwork", f"-slot={1 if stratum == 'FI' else 0}",
|
||||||
|
"-modes=export", "-gameconfig=1", "-coreworldtest=1",
|
||||||
|
"-ownerfourstrateplan=1", "-corevoxelsizecm=25", "-coreislandtest=0",
|
||||||
|
"-corefallbacktest=0", "-coreofftest=0", "-coreplantest=0",
|
||||||
|
"-settingsref=/Game/VoxelForge/DA_Settings.DA_Settings", "-corehybridmesher=0",
|
||||||
|
f"-out={output}", "-voxel.StreamingCoarseFirst=1",
|
||||||
|
"-voxel.StreamingDynamicNearReserve=1",
|
||||||
|
"-voxel.StreamingLegacyPendingResultCap=0",
|
||||||
|
"-voxel.CoreRegionConcurrentAdmission=1",
|
||||||
|
"-voxel.StreamingResultBacklogCap=256",
|
||||||
|
f"-voxel.StreamingReadyQueueDepth={ready_depth}",
|
||||||
|
f"-voxel.StreamingBackgroundWorkerCap={background_cap}",
|
||||||
|
f"-voxel.StreamingBackgroundWorkerReserve={worker_reserve}",
|
||||||
|
f"-voxel.CoreParallelForBackgroundPriority={helper_background}",
|
||||||
|
f"-voxel.StreamingTileBackgroundLow={tile_background_low}",
|
||||||
|
f"-voxel.StreamingTileBackgroundNormal={tile_background_normal}",
|
||||||
|
f"-voxel.TestLogTickFunctions={int(tick_logs)}",
|
||||||
|
"-voxel.CoreColumnCacheDirectMapped=0", "-voxel.CoreMCGradientMemoization=1",
|
||||||
|
"-voxel.StreamingCollisionLookahead=1",
|
||||||
|
f"-voxel.StreamingStartupApplyBoost={startup_boost}",
|
||||||
|
"-voxel.TestMove=1", "-voxel.TestMoveSpeedCmPerSecond=140.0",
|
||||||
|
f"-voxel.TestMoveStartSeconds={MOVE_START_S:.1f}",
|
||||||
|
f"-voxel.TestMoveDurationSeconds={MOVE_DURATION_S:.1f}",
|
||||||
|
f"-voxel.TestExitSeconds={EXIT_S:.1f}", "-voxel.TestMoveDirection=1,0,0",
|
||||||
|
"-voxel.StreamingSchedulerPolicy=3", "-voxel.TestExitOnSteady=0",
|
||||||
|
"-coreworldstreamtrace=1", f"-voxel.StartupTraceSpawnCm={spawn_cm}",
|
||||||
|
"-voxel.StartupTraceUntilEndPlay=1", "-voxel.StartupTraceThroughCrossing=1",
|
||||||
|
"-voxel.StartupTraceMaxTiles=16384", f"-voxel.StartupTraceFile={trace}",
|
||||||
|
f"-abslog={abslog}",
|
||||||
|
]
|
||||||
|
if insights:
|
||||||
|
command += ["-trace=cpu,task,frame", "-statnamedevents",
|
||||||
|
f"-tracefile={utrace}", "-tracefiletrunc"]
|
||||||
|
wrapped = ["taskset", "-c", "0-11", "timeout", f"{RUN_TIMEOUT_S}s", *command]
|
||||||
|
started = time.monotonic()
|
||||||
|
with stdout.open("w", encoding="utf-8") as stream:
|
||||||
|
process = subprocess.Popen(wrapped, cwd=ROOT, stdout=stream,
|
||||||
|
stderr=subprocess.STDOUT)
|
||||||
|
peak_rss = None
|
||||||
|
while process.poll() is None:
|
||||||
|
observed = sample_unreal_peak_rss()
|
||||||
|
if observed and (peak_rss is None or observed[0] > peak_rss[0]):
|
||||||
|
peak_rss = observed
|
||||||
|
time.sleep(0.25)
|
||||||
|
exit_code = process.wait()
|
||||||
|
observed = sample_unreal_peak_rss()
|
||||||
|
if observed and (peak_rss is None or observed[0] > peak_rss[0]):
|
||||||
|
peak_rss = observed
|
||||||
|
|
||||||
|
events = read_events(trace)
|
||||||
|
worker_backend_config = next(
|
||||||
|
(e for e in events if e.get("type") == "worker_backend_config"), {})
|
||||||
|
worker_cap_readback = next(
|
||||||
|
(e for e in events if e.get("type") == "streaming_worker_cap_readback"), {})
|
||||||
|
samples = [e for e in events if e.get("type") == "streaming_sample"]
|
||||||
|
arrival = [e for e in samples if float(e.get("t_s", 0.0)) < MOVE_START_S]
|
||||||
|
full = next((e for e in arrival
|
||||||
|
if sum(e.get("desired_by_lod", [0] * 9)[:3]) > 0
|
||||||
|
and sum(e.get("settled_by_lod", [0] * 9)[:3])
|
||||||
|
>= sum(e.get("desired_by_lod", [0] * 9)[:3])), None)
|
||||||
|
first_admission = next((e for e in events if e.get("type") == "scheduler_admission"
|
||||||
|
and e.get("desired_rebuilt") == 1), None)
|
||||||
|
request_events = [e for e in events if e.get("type") == "request"]
|
||||||
|
initial_request_events = [e for e in request_events if e.get("epoch") == 0]
|
||||||
|
tile_by_ordinal = {e.get("request_ordinal"): e for e in events
|
||||||
|
if e.get("type") == "tile"}
|
||||||
|
collision_count = sum(1 for e in events if e.get("type") == "collision_ready"
|
||||||
|
and e.get("level", 9) <= 2
|
||||||
|
and full and e.get("t_s", 1e30) <= full["t_s"])
|
||||||
|
initial_tiles = [tile_by_ordinal[r.get("request_ordinal")] for r in initial_request_events
|
||||||
|
if full and r.get("t_s", 1e30) <= full["t_s"]
|
||||||
|
and r.get("request_ordinal") in tile_by_ordinal
|
||||||
|
and tile_by_ordinal[r["request_ordinal"]].get("level", 9) <= 2]
|
||||||
|
burst_start_s = float(first_admission["t_s"]) if first_admission else -1.0
|
||||||
|
burst_end_s = float(full["t_s"]) if full else -1.0
|
||||||
|
work_by_frame = {int(e.get("frame", -1)): e for e in events
|
||||||
|
if e.get("type") == "game_thread_work_frame"}
|
||||||
|
work_values = [e for e in work_by_frame.values()
|
||||||
|
if burst_start_s <= float(e.get("t_s", -1.0)) <= burst_end_s]
|
||||||
|
arrival_frame_ids = {int(e.get("frame", -1)) for e in work_values}
|
||||||
|
scheduler_by_frame = {int(e.get("frame", -1)): e for e in events
|
||||||
|
if e.get("type") == "scheduler_admission"}
|
||||||
|
scheduler_values = [e for e in events if e.get("type") == "scheduler_admission"
|
||||||
|
and burst_start_s <= float(e.get("t_s", -1.0)) <= burst_end_s]
|
||||||
|
apply_by_frame = {int(e.get("frame", -1)): e for e in events
|
||||||
|
if e.get("type") == "apply_frame"}
|
||||||
|
commandlet_tick_by_frame = {int(e.get("frame", -1)): e for e in events
|
||||||
|
if e.get("type") == "commandlet_world_tick_frame"}
|
||||||
|
garbage_collect_by_frame = {int(e.get("frame", -1)): e for e in events
|
||||||
|
if e.get("type") == "garbage_collect"}
|
||||||
|
# apply_frame N+1 records the interval containing AVoxelWorld::Tick N. Its apply_ms
|
||||||
|
# belongs to Tick N+1, so use apply_by_frame[N] for the same-tick apply breakdown.
|
||||||
|
near_intervals = [apply_by_frame[frame + 1] for frame in arrival_frame_ids
|
||||||
|
if frame + 1 in apply_by_frame]
|
||||||
|
rmc_by_frame: dict[int, dict[str, float]] = {}
|
||||||
|
for event in events:
|
||||||
|
frame = int(event.get("frame", -1))
|
||||||
|
if event.get("type") != "rmc_submit" or frame not in arrival_frame_ids:
|
||||||
|
continue
|
||||||
|
values = rmc_by_frame.setdefault(frame, {
|
||||||
|
"component_acquire_ms": 0.0,
|
||||||
|
"create_section_group_ms": 0.0,
|
||||||
|
"create_sections_ms": 0.0,
|
||||||
|
"collision_handoff_ms": 0.0,
|
||||||
|
})
|
||||||
|
for key in values:
|
||||||
|
values[key] += float(event.get(key, 0.0))
|
||||||
|
collision_completion_by_frame: dict[int, float] = {}
|
||||||
|
for event in events:
|
||||||
|
frame = int(event.get("frame", -1))
|
||||||
|
if event.get("type") == "collision_completion" and frame in arrival_frame_ids:
|
||||||
|
collision_completion_by_frame[frame] = collision_completion_by_frame.get(frame, 0.0) \
|
||||||
|
+ float(event.get("handler_ms", 0.0))
|
||||||
|
near_requests = [r for r in request_events if full and r.get("t_s", 1e30) <= full["t_s"]
|
||||||
|
and r.get("level", 9) <= 2]
|
||||||
|
initial_lod0_counts = Counter(
|
||||||
|
tuple(r.get("tile", ())) for r in near_requests if r.get("level") == 0)
|
||||||
|
repeated_lod0_keys = [
|
||||||
|
{"tile": list(tile), "requests": count}
|
||||||
|
for tile, count in sorted(initial_lod0_counts.items()) if count > 1]
|
||||||
|
initial_scheduler_frames = [e for e in events
|
||||||
|
if e.get("type") == "scheduler_admission" and full
|
||||||
|
and e.get("t_s", 1e30) <= full["t_s"]]
|
||||||
|
worst_interval = max(near_intervals, key=lambda e: float(e.get("frame_ms", 0.0)), default=None)
|
||||||
|
worst_interval_frame_id = int(worst_interval.get("frame", -1)) if worst_interval else -1
|
||||||
|
worst_work_frame_id = worst_interval_frame_id - 1
|
||||||
|
worst_work = work_by_frame.get(worst_work_frame_id, {})
|
||||||
|
worst_apply = apply_by_frame.get(worst_work_frame_id, {})
|
||||||
|
worst_scheduler = scheduler_by_frame.get(worst_work_frame_id, {})
|
||||||
|
worst_rmc = rmc_by_frame.get(worst_work_frame_id, {})
|
||||||
|
worst_collision_completion_ms = collision_completion_by_frame.get(worst_work_frame_id, 0.0)
|
||||||
|
rmc_values = [rmc_by_frame.get(frame, {}) for frame in arrival_frame_ids]
|
||||||
|
apply_values = [apply_by_frame[frame] for frame in arrival_frame_ids
|
||||||
|
if frame in apply_by_frame]
|
||||||
|
commandlet_tick_values = [commandlet_tick_by_frame[frame] for frame in arrival_frame_ids
|
||||||
|
if frame in commandlet_tick_by_frame]
|
||||||
|
garbage_collect_values = [garbage_collect_by_frame[frame] for frame in arrival_frame_ids
|
||||||
|
if frame in garbage_collect_by_frame]
|
||||||
|
|
||||||
|
def work_stat(name: str, fraction: float | None = None) -> float | None:
|
||||||
|
values = [float(e.get(name, 0.0)) for e in work_values]
|
||||||
|
return percentile(values, fraction) if fraction is not None else max(values, default=0.0)
|
||||||
|
|
||||||
|
def scheduler_stat(name: str, fraction: float | None = None) -> float | None:
|
||||||
|
values = [float(e.get(name, 0.0)) for e in scheduler_values]
|
||||||
|
return percentile(values, fraction) if fraction is not None else max(values, default=0.0)
|
||||||
|
|
||||||
|
def rmc_stat(name: str, fraction: float | None = None) -> float | None:
|
||||||
|
values = [float(e.get(name, 0.0)) for e in rmc_values]
|
||||||
|
return percentile(values, fraction) if fraction is not None else max(values, default=0.0)
|
||||||
|
|
||||||
|
def commandlet_stat(name: str, fraction: float | None = None) -> float | None:
|
||||||
|
values = [float(e.get(name, 0.0)) for e in commandlet_tick_values]
|
||||||
|
return percentile(values, fraction) if fraction is not None else max(values, default=0.0)
|
||||||
|
|
||||||
|
row = {
|
||||||
|
"label": label, "stratum": stratum, "repeat": repeat, "phase": phase,
|
||||||
|
"ready_queue_depth": ready_depth,
|
||||||
|
"requested_background_worker_cap": background_cap,
|
||||||
|
"background_worker_cap": worker_cap_readback.get(
|
||||||
|
"streaming_background_worker_cap", background_cap),
|
||||||
|
"requested_background_worker_reserve": worker_reserve,
|
||||||
|
"background_worker_reserve": worker_backend_config.get(
|
||||||
|
"streaming_background_worker_reserve", worker_cap_readback.get(
|
||||||
|
"background_worker_reserve", worker_reserve)),
|
||||||
|
"requested_core_parallel_for_background_priority": helper_background,
|
||||||
|
"core_parallel_for_background_priority": worker_backend_config.get(
|
||||||
|
"core_parallel_for_background_priority", helper_background),
|
||||||
|
"requested_streaming_tile_background_low": tile_background_low,
|
||||||
|
"streaming_tile_background_low": worker_backend_config.get(
|
||||||
|
"streaming_tile_background_low", tile_background_low),
|
||||||
|
"requested_streaming_tile_background_normal": tile_background_normal,
|
||||||
|
"streaming_tile_background_normal": worker_backend_config.get(
|
||||||
|
"streaming_tile_background_normal", tile_background_normal),
|
||||||
|
"effective_tile_slots": worker_backend_config.get("effective_tile_slots"),
|
||||||
|
"background_worker_threads": worker_backend_config.get("taskgraph_background_threads"),
|
||||||
|
"startup_apply_boost": startup_boost, "insights": insights,
|
||||||
|
"engine_tick_logs": tick_logs,
|
||||||
|
"exit_code": exit_code,
|
||||||
|
"launch_wrapper": "taskset -c 0-11 timeout 1740s (no address-space limit)",
|
||||||
|
"peak_rss_kib": peak_rss[0] if peak_rss else None,
|
||||||
|
"peak_rss_mib": round(peak_rss[0] / 1024.0, 1) if peak_rss else None,
|
||||||
|
"peak_rss_pid": peak_rss[1] if peak_rss else None,
|
||||||
|
"endphysics_to_postphysics_gap_ms_max": None,
|
||||||
|
"endphysics_gap_frame": None,
|
||||||
|
"wall_s": round(time.monotonic() - started, 3),
|
||||||
|
"desired_t_s": next((e.get("t_s") for e in events if e.get("type") == "desired_set"), None),
|
||||||
|
"first_admission_t_s": first_admission.get("t_s") if first_admission else None,
|
||||||
|
"lod02_full_settled_t_s": full.get("t_s") if full else None,
|
||||||
|
"near_settle_from_first_admission_s": round(
|
||||||
|
float(full["t_s"]) - float(first_admission["t_s"]), 4)
|
||||||
|
if full and first_admission else None,
|
||||||
|
"desired_by_lod": full.get("desired_by_lod", [])[:3] if full else None,
|
||||||
|
"settled_by_lod": full.get("settled_by_lod", [])[:3] if full else None,
|
||||||
|
"near_admitted_tiles": len(near_requests),
|
||||||
|
"initial_lod0_admissions": sum(initial_lod0_counts.values()),
|
||||||
|
"repeated_lod0_request_keys_before_settle": repeated_lod0_keys,
|
||||||
|
"peak_result_backlog_before_settle": max(
|
||||||
|
(int(e.get("result_backlog", 0)) for e in initial_scheduler_frames), default=0),
|
||||||
|
"last_admission_t_s": max((e.get("t_s", 0.0) for e in near_requests), default=None),
|
||||||
|
"near_tile_events": len(initial_tiles),
|
||||||
|
"near_worker_seconds": round(sum(e.get("worker_task_s", 0.0) for e in initial_tiles), 4),
|
||||||
|
"near_core_seconds": round(sum(e.get("core_generate_s", 0.0) for e in initial_tiles), 4),
|
||||||
|
"collision_ready_by_settle": collision_count,
|
||||||
|
"burst_frame_count": len(near_intervals),
|
||||||
|
"burst_frame_ms_p95": round(sorted(e.get("frame_ms", 0.0) for e in near_intervals)[
|
||||||
|
min(len(near_intervals) - 1, int(.95 * (len(near_intervals) - 1)))], 3)
|
||||||
|
if near_intervals else None,
|
||||||
|
"burst_frame_ms_max": round(max((e.get("frame_ms", 0.0) for e in near_intervals), default=0.0), 3),
|
||||||
|
"burst_full_world_tick_ms_p99": round(commandlet_stat("world_tick_ms", .99) or 0.0, 3),
|
||||||
|
"burst_full_world_tick_ms_max": round(commandlet_stat("world_tick_ms") or 0.0, 3),
|
||||||
|
"burst_world_pre_actor_ms_p99": round(commandlet_stat("world_pre_actor_ms", .99) or 0.0, 3),
|
||||||
|
"burst_world_pre_actor_ms_max": round(commandlet_stat("world_pre_actor_ms") or 0.0, 3),
|
||||||
|
"burst_world_actor_phase_ms_p99": round(commandlet_stat("world_actor_phase_ms", .99) or 0.0, 3),
|
||||||
|
"burst_world_actor_phase_ms_max": round(commandlet_stat("world_actor_phase_ms") or 0.0, 3),
|
||||||
|
"burst_world_post_actor_ms_p99": round(commandlet_stat("world_post_actor_ms", .99) or 0.0, 3),
|
||||||
|
"burst_world_post_actor_ms_max": round(commandlet_stat("world_post_actor_ms") or 0.0, 3),
|
||||||
|
"burst_commandlet_readiness_check_ms_max": round(
|
||||||
|
commandlet_stat("readiness_check_ms") or 0.0, 3),
|
||||||
|
"burst_commandlet_sleep_ms_p99": round(commandlet_stat("sleep_ms", .99) or 0.0, 3),
|
||||||
|
"burst_commandlet_sleep_ms_max": round(commandlet_stat("sleep_ms") or 0.0, 3),
|
||||||
|
"burst_commandlet_loop_ms_max": round(commandlet_stat("loop_ms") or 0.0, 3),
|
||||||
|
"burst_gc_count": len(garbage_collect_values),
|
||||||
|
"burst_gc_duration_ms_max": round(max(
|
||||||
|
(float(e.get("duration_ms", 0.0)) for e in garbage_collect_values), default=0.0), 3),
|
||||||
|
"burst_max_meshes_per_frame": max((e.get("meshes_applied", 0) for e in apply_values), default=0),
|
||||||
|
"burst_max_apply_ms": round(max((e.get("apply_ms", 0.0) for e in apply_values), default=0.0), 3),
|
||||||
|
"burst_max_admit_per_frame": max((sum(e.get("submitted_by_lod", [0] * 9)[:3])
|
||||||
|
for e in scheduler_values), default=0),
|
||||||
|
"burst_frame_timing_count": len(work_values),
|
||||||
|
"burst_active_generation_workers_p95": round(work_stat("active_generation_workers", .95) or 0.0, 3),
|
||||||
|
"burst_active_generation_workers_max": round(work_stat("active_generation_workers") or 0.0, 3),
|
||||||
|
"burst_active_tile_tasks_max": round(work_stat("active_tile_tasks") or 0.0, 3),
|
||||||
|
"burst_world_tick_ms_p99": round(work_stat("tick_ms", .99) or 0.0, 3),
|
||||||
|
"burst_world_tick_ms_max": round(work_stat("tick_ms") or 0.0, 3),
|
||||||
|
"burst_update_chunks_ms_p99": round(work_stat("update_chunks_ms", .99) or 0.0, 3),
|
||||||
|
"burst_update_chunks_ms_max": round(work_stat("update_chunks_ms") or 0.0, 3),
|
||||||
|
"burst_pending_drain_ms_p99": round(work_stat("pending_drain_ms", .99) or 0.0, 3),
|
||||||
|
"burst_pending_drain_ms_max": round(work_stat("pending_drain_ms") or 0.0, 3),
|
||||||
|
"burst_post_apply_callbacks_ms_p99": round(work_stat("post_apply_callbacks_ms", .99) or 0.0, 3),
|
||||||
|
"burst_post_apply_callbacks_ms_max": round(work_stat("post_apply_callbacks_ms") or 0.0, 3),
|
||||||
|
"burst_visibility_reconcile_ms_p99": round(work_stat("visibility_reconcile_ms", .99) or 0.0, 3),
|
||||||
|
"burst_visibility_reconcile_ms_max": round(work_stat("visibility_reconcile_ms") or 0.0, 3),
|
||||||
|
"burst_collision_probe_ms_max": round(work_stat("collision_probe_ms") or 0.0, 3),
|
||||||
|
"burst_visible_apply_probe_ms_max": round(work_stat("visible_apply_probe_ms") or 0.0, 3),
|
||||||
|
"burst_lod0_ready_ms_max": round(work_stat("lod0_ready_ms") or 0.0, 3),
|
||||||
|
"burst_steady_trace_ms_max": round(work_stat("steady_trace_ms") or 0.0, 3),
|
||||||
|
"burst_modification_ms_max": round(work_stat("modification_ms") or 0.0, 3),
|
||||||
|
"burst_lifecycle_audit_ms_p99": round(work_stat("lifecycle_audit_ms", .99) or 0.0, 3),
|
||||||
|
"burst_lifecycle_audit_ms_max": round(work_stat("lifecycle_audit_ms") or 0.0, 3),
|
||||||
|
"burst_collision_gate_ms_p99": round(work_stat("collision_gate_ms", .99) or 0.0, 3),
|
||||||
|
"burst_collision_gate_ms_max": round(work_stat("collision_gate_ms") or 0.0, 3),
|
||||||
|
"burst_unload_ms_p99": round(work_stat("unload_ms", .99) or 0.0, 3),
|
||||||
|
"burst_unload_ms_max": round(work_stat("unload_ms") or 0.0, 3),
|
||||||
|
"burst_trace_json_ms_p99": round(work_stat("trace_json_ms", .99) or 0.0, 3),
|
||||||
|
"burst_trace_json_ms_max": round(work_stat("trace_json_ms") or 0.0, 3),
|
||||||
|
"burst_desired_rebuild_ms_max": round(scheduler_stat("desired_rebuild_ms") or 0.0, 3),
|
||||||
|
"burst_rmc_create_group_ms_p99": round(rmc_stat("create_section_group_ms", .99) or 0.0, 3),
|
||||||
|
"burst_rmc_create_group_ms_max": round(rmc_stat("create_section_group_ms") or 0.0, 3),
|
||||||
|
"burst_rmc_collision_handoff_ms_max": round(rmc_stat("collision_handoff_ms") or 0.0, 3),
|
||||||
|
"burst_collision_completion_ms_max": round(max(
|
||||||
|
collision_completion_by_frame.values(), default=0.0), 3),
|
||||||
|
"worst_frame_breakdown": {
|
||||||
|
"frame": worst_work_frame_id,
|
||||||
|
"world_tick_frame": worst_work_frame_id,
|
||||||
|
"frame_interval_end_frame": worst_interval_frame_id,
|
||||||
|
"frame_interval_ms": round(float(worst_interval.get("frame_ms", 0.0)), 3)
|
||||||
|
if worst_interval else None,
|
||||||
|
"apply_ms": round(float(worst_apply.get("apply_ms", 0.0)), 3)
|
||||||
|
if worst_apply else None,
|
||||||
|
"result_drain_ms": round(float(worst_apply.get("drain_ms", 0.0)), 3)
|
||||||
|
if worst_apply else None,
|
||||||
|
"outside_world_tick_ms": round(max(0.0, float(worst_interval.get("frame_ms", 0.0))
|
||||||
|
- float(worst_work.get("tick_ms", 0.0))), 3)
|
||||||
|
if worst_interval else None,
|
||||||
|
"outside_full_world_tick_ms": round(max(0.0, float(worst_interval.get("frame_ms", 0.0))
|
||||||
|
- float(commandlet_tick_by_frame.get(worst_work_frame_id, {}).get("world_tick_ms", 0.0))), 3)
|
||||||
|
if worst_interval else None,
|
||||||
|
"full_world_tick_ms": round(float(commandlet_tick_by_frame.get(
|
||||||
|
worst_work_frame_id, {}).get("world_tick_ms", 0.0)), 3),
|
||||||
|
"world_pre_actor_ms": round(float(commandlet_tick_by_frame.get(
|
||||||
|
worst_work_frame_id, {}).get("world_pre_actor_ms", 0.0)), 3),
|
||||||
|
"world_actor_phase_ms": round(float(commandlet_tick_by_frame.get(
|
||||||
|
worst_work_frame_id, {}).get("world_actor_phase_ms", 0.0)), 3),
|
||||||
|
"world_post_actor_ms": round(float(commandlet_tick_by_frame.get(
|
||||||
|
worst_work_frame_id, {}).get("world_post_actor_ms", 0.0)), 3),
|
||||||
|
"commandlet_readiness_check_ms": round(float(commandlet_tick_by_frame.get(
|
||||||
|
worst_work_frame_id, {}).get("readiness_check_ms", 0.0)), 3),
|
||||||
|
"commandlet_sleep_ms": round(float(commandlet_tick_by_frame.get(
|
||||||
|
worst_work_frame_id, {}).get("sleep_ms", 0.0)), 3),
|
||||||
|
"commandlet_loop_ms": round(float(commandlet_tick_by_frame.get(
|
||||||
|
worst_work_frame_id, {}).get("loop_ms", 0.0)), 3),
|
||||||
|
"gc_duration_ms": round(float(garbage_collect_by_frame.get(
|
||||||
|
worst_work_frame_id, {}).get("duration_ms", 0.0)), 3),
|
||||||
|
"world_tick_ms": round(float(worst_work.get("tick_ms", 0.0)), 3),
|
||||||
|
"active_generation_workers": int(worst_work.get("active_generation_workers", 0)),
|
||||||
|
"active_tile_tasks": int(worst_work.get("active_tile_tasks", 0)),
|
||||||
|
"collision_gate_ms": round(float(worst_work.get("collision_gate_ms", 0.0)), 3),
|
||||||
|
"update_chunks_ms": round(float(worst_work.get("update_chunks_ms", 0.0)), 3),
|
||||||
|
"desired_rebuild_ms": round(float(worst_scheduler.get("desired_rebuild_ms", 0.0)), 3),
|
||||||
|
"pending_drain_ms": round(float(worst_work.get("pending_drain_ms", 0.0)), 3),
|
||||||
|
"post_apply_callbacks_ms": round(float(worst_work.get("post_apply_callbacks_ms", 0.0)), 3),
|
||||||
|
"collision_probe_ms": round(float(worst_work.get("collision_probe_ms", 0.0)), 3),
|
||||||
|
"visibility_reconcile_ms": round(float(worst_work.get("visibility_reconcile_ms", 0.0)), 3),
|
||||||
|
"visible_apply_probe_ms": round(float(worst_work.get("visible_apply_probe_ms", 0.0)), 3),
|
||||||
|
"lod0_ready_ms": round(float(worst_work.get("lod0_ready_ms", 0.0)), 3),
|
||||||
|
"unload_ms": round(float(worst_work.get("unload_ms", 0.0)), 3),
|
||||||
|
"steady_trace_ms": round(float(worst_work.get("steady_trace_ms", 0.0)), 3),
|
||||||
|
"modification_ms": round(float(worst_work.get("modification_ms", 0.0)), 3),
|
||||||
|
"lifecycle_audit_ms": round(float(worst_work.get("lifecycle_audit_ms", 0.0)), 3),
|
||||||
|
"trace_json_ms": round(float(worst_work.get("trace_json_ms", 0.0)), 3),
|
||||||
|
"trace_archive_serialize_ms": round(float(
|
||||||
|
worst_work.get("trace_archive_serialize_ms", 0.0)), 3),
|
||||||
|
"collision_completion_ms": round(worst_collision_completion_ms, 3),
|
||||||
|
**{key: round(float(worst_rmc.get(key, 0.0)), 3)
|
||||||
|
for key in ("component_acquire_ms", "create_section_group_ms",
|
||||||
|
"create_sections_ms", "collision_handoff_ms")},
|
||||||
|
},
|
||||||
|
"trace": str(trace.relative_to(ROOT)), "utrace": str(utrace.relative_to(ROOT)),
|
||||||
|
"abslog": str(abslog.relative_to(ROOT)), "stdout": str(stdout.relative_to(ROOT)),
|
||||||
|
}
|
||||||
|
stdout_text = stdout.read_text(encoding="utf-8", errors="replace")
|
||||||
|
gap_ms, gap_frame = endphysics_gap(stdout)
|
||||||
|
row["endphysics_to_postphysics_gap_ms_max"] = gap_ms
|
||||||
|
row["endphysics_gap_frame"] = gap_frame
|
||||||
|
failure = re.search(r"\[VoxelForgeCoreWorldTest\] failed: ([^\r\n]+)", stdout_text)
|
||||||
|
if failure:
|
||||||
|
row["world_test_failure"] = failure.group(1)
|
||||||
|
with (OUT / "runs.jsonl").open("a", encoding="utf-8") as stream:
|
||||||
|
stream.write(json.dumps(row, sort_keys=True) + "\n")
|
||||||
|
(OUT / f"{label}.summary.json").write_text(json.dumps(row, indent=2) + "\n",
|
||||||
|
encoding="utf-8")
|
||||||
|
print("[L104UnrealArrival] " + json.dumps(row, sort_keys=True), flush=True)
|
||||||
|
if exit_code:
|
||||||
|
raise SystemExit(f"Unreal commandlet failed: {stdout}; exit={exit_code}")
|
||||||
|
if not full or len(arrival) < 8:
|
||||||
|
raise SystemExit(f"Unreal arrival trace incomplete: {stdout}; exit={exit_code}")
|
||||||
|
if insights and (not utrace.is_file() or utrace.stat().st_size < 1024):
|
||||||
|
raise SystemExit(f"Unreal Insights trace missing or empty: {utrace}")
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--stratum", choices=("FI", "SW"), required=True)
|
||||||
|
parser.add_argument("--repeat", type=int, default=1)
|
||||||
|
parser.add_argument("--phase", choices=("probe", "before", "after", "snapshot"), default="probe")
|
||||||
|
parser.add_argument("--ready-depth", type=int, default=64)
|
||||||
|
parser.add_argument("--background-cap", type=int, choices=(0, 1), default=1)
|
||||||
|
parser.add_argument("--worker-reserve", type=int, choices=(0, 1, 2, 3), default=0)
|
||||||
|
parser.add_argument("--helper-background", type=int, choices=(0, 1), default=1)
|
||||||
|
parser.add_argument("--tile-background-low", type=int, choices=(0, 1), default=1)
|
||||||
|
parser.add_argument("--tile-background-normal", type=int, choices=(0, 1), default=0)
|
||||||
|
parser.add_argument("--startup-boost", type=int, choices=(0, 1), default=1)
|
||||||
|
parser.add_argument("--tick-logs", action="store_true")
|
||||||
|
parser.add_argument("--insights", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if not EDITOR.is_file():
|
||||||
|
raise SystemExit(f"missing Unreal commandlet executable: {EDITOR}")
|
||||||
|
run(args.stratum, args.repeat, args.phase, args.ready_depth,
|
||||||
|
args.background_cap, args.worker_reserve, args.helper_background,
|
||||||
|
args.tile_background_low, args.tile_background_normal, args.startup_boost,
|
||||||
|
args.tick_logs, args.insights)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Reference in new issue
Block a user