diff --git a/Makefile b/Makefile index 0ec80817f..c51a9588f 100644 --- a/Makefile +++ b/Makefile @@ -590,7 +590,7 @@ test-cuda-integration: test-cuda-fallback: $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features test-cuda-faults \ --test cuda_fallback_tests -- --ignored --nocapture --test-threads=1 - cargo test -p lambda-vm-prover --release --features lambda-vm-prover/cuda \ + $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features lambda-vm-prover/cuda \ --test gpu_force_downgrade -- --ignored --nocapture --test-threads=1 # The prover/stark/crypto/ecsm test suite with the GPU (cuda) path enabled (requires NVIDIA diff --git a/crypto/math-cuda/src/barycentric.rs b/crypto/math-cuda/src/barycentric.rs index cce002ac0..99ba233a0 100644 --- a/crypto/math-cuda/src/barycentric.rs +++ b/crypto/math-cuda/src/barycentric.rs @@ -586,3 +586,28 @@ pub fn gather_rows_ext3_on_device( stream.synchronize()?; Ok(host) } + +#[cfg(test)] +mod tests { + use super::bary_num_chunks; + + /// Pins which of the three terms binds, per regime. Pure arithmetic — the + /// kernels' parity across chunk counts is covered by + /// `tests/barycentric_multi.rs`, which allocates a GPU. + #[test] + fn bary_num_chunks_branches() { + // Rows-bound: the domain is too short to split further, whatever the + // grid wants. 2^14/8192 = 2, under the occupancy term's 2048/100 = 20. + assert_eq!(bary_num_chunks(100, 1 << 14), 2); + // Occupancy-bound: the columns alone nearly fill the grid, so the + // domain is split less than its length would allow. 2048/256 = 8, + // under the rows term's 2^17/8192 = 16. + assert_eq!(bary_num_chunks(256, 1 << 17), 8); + // Cap-bound: at production shapes both terms clear 64 (512 and 128). + assert_eq!(bary_num_chunks(4, 1 << 20), 64); + // Degenerate inputs still yield a launchable grid (>= 1 chunk). + assert_eq!(bary_num_chunks(0, 0), 1); + assert_eq!(bary_num_chunks(usize::MAX, 1 << 20), 1); + assert_eq!(bary_num_chunks(1, 0), 1); + } +} diff --git a/crypto/math-cuda/tests/barycentric_multi.rs b/crypto/math-cuda/tests/barycentric_multi.rs index 4f58f27df..361a9c32c 100644 --- a/crypto/math-cuda/tests/barycentric_multi.rs +++ b/crypto/math-cuda/tests/barycentric_multi.rs @@ -140,14 +140,17 @@ fn run_ext3(log_trace: u32, blowup: usize, num_cols: usize, k_points: usize, see #[test] fn bary_base_multi_matches_single_point() { // Covers: k=1 degenerate, the production k=2, the kernel cap k=8, a - // single-chunk tiny n, and a column count that forces the chunk heuristic - // to its occupancy branch. + // single-chunk tiny n, a multi-chunk mid case, and the 64-chunk cap — + // the most chunks any shape can ask for, so parity is pinned at both + // ends of the chunk range. (`bary_num_chunks`'s own branch selection is + // covered by its unit tests; only the kernels are exercised here.) for (log_t, blowup, cols, k) in [ (4u32, 2usize, 3usize, 1usize), (8, 4, 10, 2), (12, 2, 5, 3), (14, 2, 100, 2), (10, 2, 4, 8), + (20, 2, 4, 2), ] { run_base(log_t, blowup, cols, k, 3000 + log_t as u64 + k as u64); } @@ -161,6 +164,7 @@ fn bary_ext3_multi_matches_single_point() { (10, 2, 3, 3), (14, 2, 40, 2), (10, 2, 4, 8), + (19, 2, 2, 2), ] { run_ext3(log_t, blowup, cols, k, 4000 + log_t as u64 + k as u64); } diff --git a/crypto/stark/src/gpu_lde.rs b/crypto/stark/src/gpu_lde.rs index 40a4b447a..73c6376a7 100644 --- a/crypto/stark/src/gpu_lde.rs +++ b/crypto/stark/src/gpu_lde.rs @@ -46,6 +46,12 @@ use crate::trace::LDETraceTable; /// measured sweep optimum on ethrex continuations (2^14 beats 2^15..2^19 and /// also beats "everything on GPU", where sub-2^14 tables lose to launch /// overhead). Override via env var for tuning. +/// +/// The same value gates the whole dispatch layer, not just the commit: R2 +/// decompose, the R3 inv-denoms/barycentric contexts, R4 DEEP and the FRI +/// fold all admit on it, so moving it moves every one of those floors +/// together. The device-only envelope is the one gate that does NOT ride on +/// it — see [`DEFAULT_DEVICE_ONLY_MIN_LDE`]. const DEFAULT_GPU_LDE_THRESHOLD: usize = 1 << 14; fn gpu_lde_threshold() -> usize { @@ -66,6 +72,15 @@ fn gpu_lde_threshold() -> usize { /// eligibility (the LOCKSTEP note below). Keep device-only to the large-table /// envelope where those paths are exercised; mid tables keep a host copy so a /// dispatch decline degrades to CPU instead of aborting. +/// +/// That degradation covers the sites that READ the LDE — they all gate on +/// `host_trace_empty()` and take their host arm. It does NOT cover the R4 +/// Merkle-proof gather: the host tree is root-only for every GPU-committed +/// table (the tree stays resident from [`DEFAULT_GPU_LDE_THRESHOLD`] upward, +/// whatever `retain_host_lde` says), so a declined `gather_proofs_dev` has +/// nothing to fall back to and aborts regardless of the host LDE. Lowering +/// the commit threshold therefore widens that one abort site even though it +/// leaves this envelope alone. const DEFAULT_DEVICE_ONLY_MIN_LDE: usize = 1 << 19; fn gpu_device_only_threshold() -> usize { @@ -2633,8 +2648,9 @@ where /// returning one [`Proof`] per position in the same order. Byte-identical to /// the host `MerkleTree::get_proof_by_pos` (guarded by the `merkle_gather` /// parity test), so R4 query openings can source proofs from the resident -/// device tree instead of the host tree. Returns `None` on any cudarc error -/// (the caller then falls back to the host tree). +/// device tree instead of the host tree. Returns `None` on any cudarc error — +/// which every caller treats as a hard abort, NOT a fallback: a resident tree +/// leaves the host tree root-only, so there is no host path to walk. pub(crate) fn gather_proofs_dev( tree: &math_cuda::lde::GpuMerkleTree, positions: &[usize], @@ -3162,7 +3178,8 @@ mod split_tree_tests { /// isolates the tree layout/hashing under test. #[test] fn split_trees_match_cpu_subset_commits() { - // Above the dispatch threshold (2^19 LDE) so the GPU path must engage. + // This shape's LDE is 2^19, well above the dispatch threshold, so the + // GPU path must engage. let n: usize = 1 << 18; let blowup: usize = 2; let m: usize = 5; diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index adf0867c3..0da600b52 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -2687,9 +2687,11 @@ pub trait IsStarkProver< !lde_trace.host_trace_empty(), "R4 {what} opening fell back to the host tree, but it is device-only (empty)" ); - // A root-only host tree means the nodes are device-resident: the - // host walk would emit an empty path for position 0 instead of - // failing, so a broken proofs↔tree pairing must abort here. + // A root-only host tree means the nodes are device-resident, so a + // broken proofs↔tree pairing must abort here. `get_proof_by_pos` + // already refuses a root-only tree, but the panic it produces + // downstream reads "FRI query index in bounds" — this names the + // real cause instead. assert!( !tree.is_root_only(), "R4 {what} opening fell back to a root-only host tree (nodes device-resident)" diff --git a/crypto/stark/src/trace.rs b/crypto/stark/src/trace.rs index 6c3d1cdc0..9a989b29b 100644 --- a/crypto/stark/src/trace.rs +++ b/crypto/stark/src/trace.rs @@ -54,8 +54,10 @@ where } /// Device-resident row-major main trace, pre-uploaded ahead of the prove. -/// Excluded from logical trace equality and opaque in `Debug`, matching -/// [`ResidentMainTrace`]. +/// Opaque in `Debug` like [`ResidentMainTrace`], and fully excluded from +/// logical trace equality: this is a cache of data the host trace still owns, +/// so two traces that differ only here are equal. (`ResidentMainTrace` still +/// compares its row count, because it can be the sole owner of the data.) #[cfg(feature = "cuda")] #[derive(Clone)] pub(crate) struct PreUploadedMainTrace { diff --git a/scripts/profiling/README.md b/scripts/profiling/README.md index f4ad4d57b..1285a6caa 100644 --- a/scripts/profiling/README.md +++ b/scripts/profiling/README.md @@ -104,6 +104,7 @@ the phase that enqueued them even if they execute later. | `capture_env.sh` | env JSON to stdout — attach to anything you measure by hand | | `phase_table.py [--util u.csv]… tl.json…` | aggregate timelines; `--instances LABEL` adds per-instance tables for deeper repeated spans, `--min-pct X` hides noise rows | | `nsys_phase_busy.py report.sqlite [--top N]` | the GPU busy report from `nsys export --type sqlite` | +| `h2d_histo.py report.sqlite` | H2D/D2H bytes grouped by (phase, innermost NVTX range, transfer size) — names the dominant uploaders inside a phase. Prints the top 20 per direction | | `nvml_sampler.py -o out.csv [-i 0.1]` | standalone 10 Hz GPU util sampler (epoch-ns timestamps, aligns with span `start_ns`) | | `timeline_to_perfetto.py tl.json > trace.json` | span tree for ui.perfetto.dev | @@ -122,6 +123,15 @@ Useful prover knobs for A/B experiments (pre-existing, see plan §11): `LAMBDA_VM_GPU_BARY_THRESHOLD`, `LAMBDA_VM_VRAM_BUDGET_MB`, `TABLE_PARALLELISM`. +Residency and diagnostic knobs: + +| var | effect | +|---|---| +| `LAMBDA_VM_GPU_DEVICE_ONLY_THRESHOLD=` | minimum LDE size for the device-only envelope (default 2^19), independent of `LAMBDA_VM_GPU_LDE_THRESHOLD`. Raise it to shed device-only tables without giving up GPU commits — a finer instrument than `LAMBDA_VM_DISABLE_DEVICE_ONLY=1` | +| `LAMBDA_VM_TRACE_PREUPLOAD_MB=` | budget for pre-uploading the epoch's biggest main traces from the builder thread, so R1 commits D2D-copy instead of paying their H2D. Default 0 (off); capped at a quarter of the device VRAM budget. Wall-neutral on a 5090 and it competes with the prove peak on small cards, so it is for PCIe-bound setups | +| `LAMBDA_VM_GPU_FORCE_DOWNGRADE=1` | test hook: decline the device R2 path unconditionally, so every device-only table exercises the host recovery. Used by the `gpu_force_downgrade` test | +| `LAMBDA_VM_GPU_XCHECK=1` | after each table, re-run the verifier's composition consistency check in-process; on a mismatch, recompute each device stage on host, report which one diverged, and abort. For localizing silent device-side corruption | + ## Continuations: per-epoch data for parallelization `prove_continuation` is instrumented independently of the monolithic path diff --git a/scripts/profiling/h2d_histo.py b/scripts/profiling/h2d_histo.py index 1317cc103..68047e0b9 100644 --- a/scripts/profiling/h2d_histo.py +++ b/scripts/profiling/h2d_histo.py @@ -33,7 +33,7 @@ def main(): api = load_api_calls(con, tset) chain_at = build_range_lookup(nvtx) - def chain_for(corr, start): + def chain_for(corr): if corr in api: api_start, tid = api[corr] c = chain_at(tid, api_start) @@ -55,7 +55,7 @@ def innermost(chain): for start, end, kind, nbytes, corr in memcpys: if kind not in ("h2d", "d2h"): continue - chain = chain_for(corr, start) + chain = chain_for(corr) key = (kind, coarse_of(chain), innermost(chain), nbytes) h = hist[key] h[0] += 1