diff --git a/Standalone/Core/src/ShapeWorldInternal.h b/Standalone/Core/src/ShapeWorldInternal.h index 2099142..57f5efa 100644 --- a/Standalone/Core/src/ShapeWorldInternal.h +++ b/Standalone/Core/src/ShapeWorldInternal.h @@ -31,6 +31,19 @@ #include #include +// L86-only ceiling measurement. This removes passage carving/support and the transition-mouth +// passage fade so timing can measure the best possible passage-representation gain. Never ship +// or expose this diagnostic as a world setting. +static bool VF_IsL86PassageCompositionDisabled() +{ + static const bool bDisabled = []() + { + const char* Value = std::getenv("VOXELFORGE_L86_DISABLE_PASSAGE_COMPOSITION"); + return Value && Value[0] == '1' && Value[1] == '\0'; + }(); + return bDisabled; +} + // Standalone-only world compiler. It compiles the runtime room/tunnel/passages into // axis-line intervals, composes rock/cave signs into packed Z-column words, then displaces // only the conservative surface band. The copied Unreal generator remains the A/B reference. @@ -195,19 +208,26 @@ public: { mayContainResolvedBoundaryBand_=shape.ResolvedBoundaryBandsIntersectSubvoxelRange( minimumSubvoxels_[2],maximumSubvoxels_[2]); - mayContainTransitionPassageFade_=shape.ResolvedTransitionPassageFadeIntersectsTileXY( - minimumSubvoxels_,maximumSubvoxels_); - featureIndices_=shape.BuildPassageFeatureTileList( - minimumSubvoxels_,maximumSubvoxels_); + const bool disablePassageComposition = VF_IsL86PassageCompositionDisabled(); + mayContainTransitionPassageFade_ = !disablePassageComposition + && shape.ResolvedTransitionPassageFadeIntersectsTileXY( + minimumSubvoxels_, maximumSubvoxels_); + if (!disablePassageComposition) + { + featureIndices_ = shape.BuildPassageFeatureTileList( + minimumSubvoxels_, maximumSubvoxels_); + } stampIndices_=shape.BuildStampTileList(minimumSubvoxels_,maximumSubvoxels_); cavernFeatureIndices_=shape.BuildCavernFeatureTileList( minimumSubvoxels_,maximumSubvoxels_); - passageRoutes_.assign(shape.ResolvedLayout?shape.ResolvedLayout->passages.size():0,0); + passageRoutes_.assign(shape.ResolvedLayout && !disablePassageComposition + ? shape.ResolvedLayout->passages.size() : 0, 0); passageColumns_=passageRoutes_; passageMouthSupports_=passageRoutes_; passageMouthSupportSegments_.resize(passageRoutes_.size()); passageSegmentRangeCaches_.resize(passageRoutes_.size()); - if(shape.ResolvedLayout&&!shape.ResolvedLayout->passages.empty()) + if(!disablePassageComposition&&shape.ResolvedLayout + &&!shape.ResolvedLayout->passages.empty()) { constexpr double Q=static_cast( VoxelForgeCore::WorldAssemblySubvoxelsPerVoxel); @@ -238,11 +258,14 @@ public: upperMouthSupportPassages_.push_back(passageIndex); } } - for(const auto& crossing:shape.PassageVoidCrossings) - if(crossing.needsColumn - &&crossing.bottomZSubvoxels<=maximumSubvoxels_[2] - &&crossing.topZSubvoxels>=minimumSubvoxels_[2]) - columnZOverlap_=true; + if(!disablePassageComposition) + { + for(const auto& crossing:shape.PassageVoidCrossings) + if(crossing.needsColumn + &&crossing.bottomZSubvoxels<=maximumSubvoxels_[2] + &&crossing.topZSubvoxels>=minimumSubvoxels_[2]) + columnZOverlap_=true; + } for(const std::uint32_t featureIndex:featureIndices_) { if(featureIndex>=shape.PassageTileFeatures.size())continue; @@ -2233,7 +2256,7 @@ public: // mouth that crosses a wide transition band. Keep that opt-in field whole; // its host-rock value already supplies the ceiling and bedrock. float boundary=graphDensity; - if(!nativeLattice) + if(!nativeLattice && !VF_IsL86PassageCompositionDisabled()) { VoxelForgeCore::Linux77Profiler::FScopedPhase l77BandsPhase( VoxelForgeCore::Linux77Profiler::EPhase::Bands); diff --git a/Standalone/Research/WorldAssemblerM3/Linux86/REPORT.md b/Standalone/Research/WorldAssemblerM3/Linux86/REPORT.md new file mode 100644 index 0000000..0abb74d --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux86/REPORT.md @@ -0,0 +1,28 @@ +# Linux 86 — passage ceiling and cache falsifier + +## Passage-disabled ceiling + +L80 protocol: one warmup per stratum and worker count, followed by three interleaved W8/W12 pairs. Values are median seconds to LOD0–2 ready / whole view ready. The diagnostic removes route carving, support composition, transition-mouth passage fade, and per-tile passage feature lists. + +| Stratum | W8 | W12 | +|---|---:|---:| +| SurfaceWorld | 1.008 / 2.194 | 1.122 / 2.253 | +| FloatingIslands | 2.343 / 3.589 | 2.179 / 3.337 | +| TunnelNetwork | 2.343 / 3.451 | 2.153 / 3.151 | +| VerticalShafts | 1.327 / 2.289 | 1.123 / 2.026 | + +The FI/TN ceiling remains more than two seconds. In representative W12 profiles without passage composition, marching-cubes grid fill used 15.7 worker-seconds for FI and 14.8 for TN across LOD0–2; skirts used another 0.72 worker-seconds in each. The mesh grid work is the main remaining cost. Shape compilation was about 0.24–0.25 s wall time. + +Logs and raw runs are in `Saved/Linux/l86/ceiling/`. + +## Prototype and decision + +I tried a sparse, lazy world-coordinate map keyed by the exact float coordinate and incoming density bits. A miss ran the existing full passage composer and stored its exact post-composition output; a repeat became a lookup. This preserves carve ordering, support restoration, deferred support, and floors without interpolation. Miss computation ran inside tile generation, so it overlapped generation and remained in cold readiness. + +One FI W8 falsification smoke took **3.353 / 5.621 s**, versus L85’s normal-field W8 medians of **2.286 / 3.469 s** (+47% / +62%). It recorded 4.67M hits and 14.57M misses (24.3% hits), about **699 MB** estimated map storage, and **9.996 worker-seconds** of miss computation. That CPU time ran inside tile generation; the measured 5.621 s whole-view readiness includes it. I stopped at this clear regression; the three-pair W8/W12 prototype matrix was not run. Smoke log: `Saved/Linux/l86/logs/cache-smoke-fi-w8.log`. + +The result cache prototype was removed from source. It used no interpolation, but max/mean displacement was not measured. Post-prototype support, capsule, grade, ripple, naturalness, mouth-profile, and three-interface audits; before/after captures; Clang/GCC peer determinism; and fast tier were not run after the speed gate failed. The exact-key design was not treated as a substitute for those gates. No hashes were repinned or version bumped. + +## Files and failed runs + +The only retained Core change is the L86 environment-only ceiling hook in `Standalone/Core/src/ShapeWorldInternal.h`; it is diagnostic and not a world setting. No `Source/` files changed, so no Unreal build was needed. The cache smoke exited successfully but failed the speed gate. No run exceeded 30 minutes. No commit, push, or stash was made. diff --git a/Standalone/Research/WorldAssemblerM3/Linux86/RunL86CacheScaling.py b/Standalone/Research/WorldAssemblerM3/Linux86/RunL86CacheScaling.py new file mode 100644 index 0000000..e0821b8 --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux86/RunL86CacheScaling.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Interleaved W8/W12 cold-readiness comparison for L86 exact passage result reuse.""" +from __future__ import annotations + +import json +import re +import statistics +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l86/cache-scaling" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l86/bin/VoxelForgeCoreHeadless-clang19" +WORKERS = (8, 12) +MODES = ("direct", "cache") +CAMERAS = { + 0: (8.0, 16.0, -372.2626953125), + 1: (25.8369140625, 3.19921875, -737.0), + 2: (19.166015625, -14.2001953125, -753.3740234375), + 3: (-20.0625, -5.068359375, -900.353515625), +} +EXPECTED_NEAR = {0: 1408, 1: 1216, 2: 1344, 3: 1344} +EXPECTED_WHOLE = {0: 2850, 1: 2249, 2: 2233, 3: 2329} +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") +NEAR = re.compile( + r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", + re.M, +) +CACHE = re.compile( + r"^\[L86PassageResultCache\] hits=(\d+) misses=(\d+) entries=(\d+) " + r"estimated_bytes=(\d+) exact_bake_ms=([0-9.]+) bake_mode=(\S+)$", re.M) + + +def run(strate: int, workers: int, mode: str, phase: str, replicate: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + label = f"s{strate}-w{workers}-{mode}-{phase}{replicate}" + log = OUT / f"{label}.log" + camera = ",".join(f"{value:.12f}" for value in CAMERAS[strate]) + ",0,0" + cache_value = "1" if mode == "cache" else "0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", + "env", "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", + f"VOXELFORGE_L86_PASSAGE_RESULT_CACHE={cache_value}", "timeout", "1740s", + str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark", + "--stream-pipeline", f"--workers={workers}", "--clip-radius=3", + "--max-clip-level=8", "--max-level=8", f"--camera={camera}", + "--profile-lod-work", "--stream-scheduler-policy=3", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + output = log.read_text(encoding="utf-8", errors="replace") + pipeline_match, near_match = PIPELINE.search(output), NEAR.search(output) + if result.returncode or pipeline_match is None or near_match is None: + raise RuntimeError(f"cache scaling run failed: exit={result.returncode} {log}") + near_done, near_total = int(near_match.group(1)), int(near_match.group(2)) + if (near_done, near_total) != (EXPECTED_NEAR[strate], EXPECTED_NEAR[strate]): + raise RuntimeError(f"incomplete near set in {log}: {near_done}/{near_total}") + fields = dict(FIELDS.findall(pipeline_match.group(1))) + whole_done, whole_total = (int(value) for value in fields["whole_tiles"].split("/")) + if (whole_done, whole_total) != (EXPECTED_WHOLE[strate], EXPECTED_WHOLE[strate]): + raise RuntimeError(f"incomplete whole set in {log}: {whole_done}/{whole_total}") + cache_rows = CACHE.findall(output) + if mode == "cache" and not cache_rows: + raise RuntimeError(f"cache mode produced no cache statistics: {log}") + cache_totals = { + "hits": sum(int(row[0]) for row in cache_rows), + "misses": sum(int(row[1]) for row in cache_rows), + "entries": sum(int(row[2]) for row in cache_rows), + "estimated_bytes": sum(int(row[3]) for row in cache_rows), + "exact_bake_ms": sum(float(row[4]) for row in cache_rows), + "bake_mode": sorted({row[5] for row in cache_rows}), + } + row = { + "strate": strate, "workers": workers, "mode": mode, + "phase": phase, "replicate": replicate, + "exit_code": result.returncode, "wall_s": round(time.monotonic() - started, 3), + "lod02_ready_s": float(near_match.group(3)), + "whole_view_ready_s": float(fields["whole_view_ready"].removesuffix("s")), + "shape_s": float(fields["shape"].removesuffix("s")), + "far_shape_s": float(fields["far_shape"].removesuffix("s")), + "near_tiles": near_total, "whole_tiles": whole_total, + "cache": cache_totals, "log": str(log.relative_to(ROOT)), + } + print("[L86CacheScalingRun] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + if not BINARY.is_file() or not DATA.is_file(): + raise SystemExit(f"missing input: {BINARY} or {DATA}") + rows = [] + for strate in range(4): + for workers in WORKERS: + for mode in MODES: + run(strate, workers, mode, "warmup", 0) + worker_orders = ((8, 12), (12, 8), (8, 12)) + mode_orders = (("direct", "cache"), ("cache", "direct"), + ("direct", "cache")) + for replicate, (worker_order, mode_order) in enumerate( + zip(worker_orders, mode_orders), 1): + for workers in worker_order: + for mode in mode_order: + rows.append(run(strate, workers, mode, "measured", replicate)) + (OUT / "runs-partial.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + summary = [] + for strate in range(4): + for workers in WORKERS: + entry = {"strate": strate, "workers": workers, "measured_runs": 3} + for mode in MODES: + subset = [row for row in rows if row["strate"] == strate + and row["workers"] == workers and row["mode"] == mode] + for metric in ("lod02_ready_s", "whole_view_ready_s", "shape_s", "far_shape_s"): + values = [row[metric] for row in subset] + entry[f"{mode}_{metric}"] = values + entry[f"{mode}_{metric}_median"] = statistics.median(values) + entry[f"{mode}_{metric}_min"] = min(values) + entry[f"{mode}_{metric}_max"] = max(values) + entry[f"{mode}_cache"] = [row["cache"] for row in subset] + summary.append(entry) + (OUT / "summary.json").write_text( + json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Standalone/Research/WorldAssemblerM3/Linux86/RunL86Ceiling.py b/Standalone/Research/WorldAssemblerM3/Linux86/RunL86Ceiling.py new file mode 100644 index 0000000..a1bfd3a --- /dev/null +++ b/Standalone/Research/WorldAssemblerM3/Linux86/RunL86Ceiling.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Measure the passage-disabled best-case ceiling using the L80 interleaved protocol.""" +from __future__ import annotations + +import json +import re +import statistics +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +OUT = ROOT / "Saved/Linux/l86/ceiling" +DATA = ROOT / "Standalone/data/owner_world.json" +BINARY = ROOT / "Saved/Linux/l86/bin/VoxelForgeCoreHeadless-clang19" +WORKERS = (8, 12) +CAMERAS = { + 0: (8.0, 16.0, -372.2626953125), + 1: (25.8369140625, 3.19921875, -737.0), + 2: (19.166015625, -14.2001953125, -753.3740234375), + 3: (-20.0625, -5.068359375, -900.353515625), +} +EXPECTED_NEAR = {0: 1408, 1: 1216, 2: 1344, 3: 1344} +EXPECTED_WHOLE = {0: 2850, 1: 2249, 2: 2233, 3: 2329} +PIPELINE = re.compile(r"^\[StreamPipeline\] (.*)$", re.M) +FIELDS = re.compile(r"([a-zA-Z0-9_]+)=([^ ]+)") +NEAR = re.compile( + r"^\[WorldAssemblerNearLODReady\] lods=0-2 tiles=(\d+)/(\d+) cold_ready_s=([0-9.]+)$", + re.M, +) + + +def run(strate: int, workers: int, phase: str, replicate: int) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + label = f"s{strate}-w{workers}-{phase}{replicate}" + log = OUT / f"{label}.log" + camera = ",".join(f"{value:.12f}" for value in CAMERAS[strate]) + ",0,0" + command = [ + "taskset", "-c", "0-11", "prlimit", "--as=6442450944", "--", + "env", "VOXELFORGE_L81_PASSAGE_BOUND_PADDING_VOXELS=0", + "VOXELFORGE_L86_DISABLE_PASSAGE_COMPOSITION=1", "timeout", "1740s", + str(BINARY), str(DATA), "--world-assembler-prototype", "--stream-benchmark", + "--stream-pipeline", f"--workers={workers}", "--clip-radius=3", + "--max-clip-level=8", "--max-level=8", f"--camera={camera}", + "--profile-lod-work", "--stream-scheduler-policy=3", + ] + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, timeout=1740) + output = log.read_text(encoding="utf-8", errors="replace") + pipeline_match, near_match = PIPELINE.search(output), NEAR.search(output) + if result.returncode or pipeline_match is None or near_match is None: + raise RuntimeError(f"ceiling run failed: exit={result.returncode} {log}") + near_done, near_total = int(near_match.group(1)), int(near_match.group(2)) + if (near_done, near_total) != (EXPECTED_NEAR[strate], EXPECTED_NEAR[strate]): + raise RuntimeError(f"incomplete near set in {log}: {near_done}/{near_total}") + fields = dict(FIELDS.findall(pipeline_match.group(1))) + whole_done, whole_total = (int(value) for value in fields["whole_tiles"].split("/")) + if (whole_done, whole_total) != (EXPECTED_WHOLE[strate], EXPECTED_WHOLE[strate]): + raise RuntimeError(f"incomplete whole set in {log}: {whole_done}/{whole_total}") + row = { + "strate": strate, "workers": workers, "phase": phase, "replicate": replicate, + "exit_code": result.returncode, "wall_s": round(time.monotonic() - started, 3), + "lod02_ready_s": float(near_match.group(3)), + "whole_view_ready_s": float(fields["whole_view_ready"].removesuffix("s")), + "shape_s": float(fields["shape"].removesuffix("s")), + "far_shape_s": float(fields["far_shape"].removesuffix("s")), + "first_tile_ready_s": float(fields["first_tile_ready"].removesuffix("s")), + "neighborhood_ready_s": float(fields["neighborhood_ready"].removesuffix("s")), + "full_detail_ready_s": float(fields["full_detail_ready"].removesuffix("s")), + "near_tiles": near_total, "whole_tiles": whole_total, + "log": str(log.relative_to(ROOT)), + } + print("[L86CeilingRun] " + json.dumps(row, sort_keys=True), flush=True) + return row + + +def main() -> int: + if not BINARY.is_file() or not DATA.is_file(): + raise SystemExit(f"missing input: {BINARY} or {DATA}") + rows = [] + for strate in range(4): + for workers in WORKERS: + run(strate, workers, "warmup", 0) + for replicate, order in enumerate(((8, 12), (12, 8), (8, 12)), 1): + for workers in order: + rows.append(run(strate, workers, "measured", replicate)) + (OUT / "runs-partial.json").write_text( + json.dumps(rows, indent=2) + "\n", encoding="utf-8") + summary = [] + for strate in range(4): + for workers in WORKERS: + subset = [row for row in rows + if row["strate"] == strate and row["workers"] == workers] + item = {"strate": strate, "workers": workers, "measured_runs": len(subset)} + for metric in ( + "lod02_ready_s", "whole_view_ready_s", "shape_s", "far_shape_s", + "first_tile_ready_s", "neighborhood_ready_s", "full_detail_ready_s", + ): + values = [row[metric] for row in subset] + item[metric] = values + item[f"{metric}_median"] = statistics.median(values) + item[f"{metric}_min"] = min(values) + item[f"{metric}_max"] = max(values) + summary.append(item) + (OUT / "summary.json").write_text( + json.dumps({"runs": rows, "summary": summary}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())