From 504034357e6764a72795e3601ddb574812376d51 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:26:27 +0100 Subject: [PATCH 001/163] mason: add sibling revision allocation probe Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 41dd7ac91bb4b88fa40fb4d6adec5e121065a0cd) (cherry picked from commit 3f127c1948a1324ad8d3cc0968359686eb43174f) --- crates/aft/examples/alloc_trace_probe.rs | 26 +++++++++++++++++++++++- crates/aft/src/data_layer/checkout.rs | 3 ++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/aft/examples/alloc_trace_probe.rs b/crates/aft/examples/alloc_trace_probe.rs index 503b8693b0..6ea91abcaf 100644 --- a/crates/aft/examples/alloc_trace_probe.rs +++ b/crates/aft/examples/alloc_trace_probe.rs @@ -12,6 +12,8 @@ //! restart [threads] //! cargo run --profile ci-storm -p agent-file-tools --example alloc_trace_probe -- \ //! replay [publish_every|daemon] [threads] +//! cargo run --profile ci-storm -p agent-file-tools --example alloc_trace_probe -- \ +//! transition [threads] //! ``` //! //! `overhead` runs two workloads in one process and prints one JSON line @@ -744,6 +746,7 @@ fn restart(mut args: impl Iterator) { } fn main() { + let _ = env_logger::try_init(); let mut args = std::env::args().skip(1); match args.next().as_deref() { Some("overhead") => overhead(args), @@ -751,6 +754,27 @@ fn main() { Some("refresh") => refresh(args), Some("restart") => restart(args), Some("replay") => replay(args), - _ => panic!("usage: alloc_trace_probe overhead|burst|refresh|restart|replay ..."), + Some("transition") => transition(args), + _ => panic!("usage: alloc_trace_probe overhead|burst|refresh|restart|replay|transition ..."), } } + +/// Measures a new checkout whose revision differs from its published sibling. +/// The supplied roots stay intact; the store is private to this run. +fn transition(mut args: impl Iterator) { + let base = std::fs::canonicalize(args.next().unwrap()).unwrap(); + let target = std::fs::canonicalize(args.next().unwrap()).unwrap(); + let threads = args.next().map_or(8, |threads| threads.parse().unwrap()); + start_tracing(); + let scratch = tempfile::tempdir().unwrap(); + let store = scratch.path().join("store"); + let family = FamilyId::from_digest(Digest128::of(b"alloc trace transition")); + let layer = Arc::new(DataLayer::new(256 << 20, threads)); + let sibling = Checkout::bind(Arc::clone(&layer), &store, family, &base).unwrap(); + phase("base_build", || format!("{:?}", sibling.build().unwrap())); + let worktree = Checkout::bind(Arc::clone(&layer), &store, family, &target).unwrap(); + phase("target_build", || { + with_cache_counts(&layer, || format!("{:?}", worktree.build().unwrap())) + }); + println!("engine parts {:?}", worktree.engine_memory_parts()); +} diff --git a/crates/aft/src/data_layer/checkout.rs b/crates/aft/src/data_layer/checkout.rs index 9354b01c32..98518988b1 100644 --- a/crates/aft/src/data_layer/checkout.rs +++ b/crates/aft/src/data_layer/checkout.rs @@ -1861,7 +1861,8 @@ impl Checkout { let text_ready_ms = elapsed_ms(build_started); checkpoint()?; let started = Instant::now(); - self.refresh_callgraph()?; + let update = self.refresh_callgraph()?; + log::debug!("sibling callgraph update: {update:?}"); let resolve_ms = elapsed_ms(started); let started = Instant::now(); let delta = self.publish_changes_as(true)?; From ae7c46214ea3a17980d8ee72ce6612c7b9067728 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:37:36 +0100 Subject: [PATCH 002/163] mason: measure refresh stage ownership Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 6d10766403ffcd4a96128876af0883e5f0193bfc) (cherry picked from commit 9ea44d9c86a6f90676e8200377ae133b35dd50da) --- crates/aft/src/data_layer/callgraph.rs | 16 +++++++++++++++ crates/aft/src/data_layer/incremental.rs | 25 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/crates/aft/src/data_layer/callgraph.rs b/crates/aft/src/data_layer/callgraph.rs index cd383a80dd..a2523ac8f4 100644 --- a/crates/aft/src/data_layer/callgraph.rs +++ b/crates/aft/src/data_layer/callgraph.rs @@ -592,6 +592,22 @@ impl EncodedResolution { } impl Resolutions { + pub(crate) fn profile_parts(&self) -> [usize; 5] { + use aft_store::heap::HeapBytes as _; + let mut seen = aft_store::heap::Seen::new(); + let mut parts = [0; 5]; + for slot in self.map.values() { + if let Some(Some(file)) = slot.value.get() { + parts[0] += file.rows.heap_bytes(&mut seen); + parts[1] += file.binding.heap_bytes(&mut seen); + parts[2] += file.dispatch.heap_bytes(&mut seen); + parts[3] += file.sites.heap_bytes(&mut seen); + parts[4] += file.site_dependencies.heap_bytes(&mut seen); + } + } + parts + } + /// Every file `stored` has a resolution for, each decoded on first read. pub(crate) fn stored(stored: &Arc) -> Self { Self { diff --git a/crates/aft/src/data_layer/incremental.rs b/crates/aft/src/data_layer/incremental.rs index 2543832e05..1226095723 100644 --- a/crates/aft/src/data_layer/incremental.rs +++ b/crates/aft/src/data_layer/incremental.rs @@ -714,6 +714,8 @@ impl LiveCheckout { ..UpdateStats::default() }; let old = std::mem::replace(&mut self.inputs, next); + let mut profile_clock = Instant::now(); + self.profile_refresh("start", &mut profile_clock); let inputs = &self.inputs; let membership = changed .iter() @@ -864,6 +866,7 @@ impl LiveCheckout { stats.answers_checked = stale.checked; stats.answers_changed = stale.changed; stats.facts_ms = started.elapsed().as_secs_f64() * 1000.0; + self.profile_refresh("facts", &mut profile_clock); // Pass 1. A binding depends on the file's own parse, on which paths // exist, and on configuration it read. @@ -986,6 +989,7 @@ impl LiveCheckout { self.bound.set(path, bound); } stats.bind_ms = started.elapsed().as_secs_f64() * 1000.0; + self.profile_refresh("bind", &mut profile_clock); // What a recorded answer can depend on that changed. let mut changed_keys = surface_keys.clone(); @@ -1092,6 +1096,7 @@ impl LiveCheckout { } drop(answers); drop(trace_answers); + self.profile_refresh("replay", &mut profile_clock); // Files kept whose binding was redone carry their resolution over. for path in &rebind { if resolve.contains(path) { @@ -1132,6 +1137,7 @@ impl LiveCheckout { self.resolved.insert(path, file); } stats.resolve_ms = started.elapsed().as_secs_f64() * 1000.0; + self.profile_refresh("resolve", &mut profile_clock); // Pass 3, for files whose rows were kept: each trace was replayed // with its file's surface answers above. Resolving files again @@ -1174,6 +1180,7 @@ impl LiveCheckout { } drop(index); drop(facts); + self.profile_refresh("dispatch", &mut profile_clock); // Pass 4. let started = Instant::now(); @@ -1212,6 +1219,7 @@ impl LiveCheckout { } stats.sites_resolved = paths.len(); stats.sites_ms = started.elapsed().as_secs_f64() * 1000.0; + self.profile_refresh("sites", &mut profile_clock); stats.touched = rebind .into_iter() .chain(resolve) @@ -1221,6 +1229,23 @@ impl LiveCheckout { .collect(); Ok(stats) } + + fn profile_refresh(&self, stage: &str, clock: &mut Instant) { + if std::env::var_os("AFT_REFRESH_PROFILE").is_none() { + return; + } + let wall_ms = clock.elapsed().as_millis(); + let cpu = std::fs::read_to_string("/proc/self/stat") + .ok() + .and_then(|line| { + let rest = line.get(line.rfind(')')? + 2..)?; + let fields = rest.split_whitespace().collect::>(); + Some(fields.get(11)?.parse::().ok()? + fields.get(12)?.parse::().ok()?) + }); + let parts = self.memory_parts(&mut aft_store::heap::Seen::new()); + log::info!("refresh profile {stage}: wall_ms={wall_ms} cpu_ticks={cpu:?} decoded_parses={} decoded_resolutions={} parts={parts:?} resolution_parts={:?}", self.inputs.parses.decoded_count(), self.resolved.decoded_count(), self.resolved.profile_parts()); + *clock = Instant::now(); + } } aft_store::heap_bytes_of_fields!(Readers { From 9e50aa9cb69f690b9cd48bff4c2efec98b9e6bfe Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:53:29 +0100 Subject: [PATCH 003/163] mason: expose test-only refresh phase measurements Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit fb0df3e0cec63693685c0900ced4cb6479f37904) (cherry picked from commit e81e75a602c28b83dfc7159a41a91e21c2641db0) --- crates/aft/examples/alloc_trace_probe.rs | 169 +++++++++++++++++++-- crates/aft/src/data_layer/callgraph.rs | 7 + crates/aft/src/data_layer/checkout.rs | 13 ++ crates/aft/src/data_layer/incremental.rs | 23 +++ crates/aft/src/data_layer/mod.rs | 2 + crates/aft/src/data_layer/refresh_probe.rs | 16 ++ 6 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 crates/aft/src/data_layer/refresh_probe.rs diff --git a/crates/aft/examples/alloc_trace_probe.rs b/crates/aft/examples/alloc_trace_probe.rs index 6ea91abcaf..cddf423929 100644 --- a/crates/aft/examples/alloc_trace_probe.rs +++ b/crates/aft/examples/alloc_trace_probe.rs @@ -13,7 +13,7 @@ //! cargo run --profile ci-storm -p agent-file-tools --example alloc_trace_probe -- \ //! replay [publish_every|daemon] [threads] //! cargo run --profile ci-storm -p agent-file-tools --example alloc_trace_probe -- \ -//! transition [threads] +//! transition [threads] [sibling|refresh|cold] //! ``` //! //! `overhead` runs two workloads in one process and prints one JSON line @@ -61,8 +61,8 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use aft::alloc_trace::TracingAllocator; @@ -75,25 +75,45 @@ use aft_store::ids::{Digest128, FamilyId}; struct Switch; static TRACE: AtomicBool = AtomicBool::new(false); +static COUNTING: AtomicBool = AtomicBool::new(false); +static LIVE: AtomicIsize = AtomicIsize::new(0); +static PEAK: AtomicIsize = AtomicIsize::new(0); +static STAGE_RSS: AtomicU64 = AtomicU64::new(0); + +fn count(bytes: isize) { + if COUNTING.load(Ordering::Relaxed) { + let live = LIVE.fetch_add(bytes, Ordering::Relaxed) + bytes; + PEAK.fetch_max(live, Ordering::Relaxed); + } +} unsafe impl GlobalAlloc for Switch { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - if TRACE.load(Ordering::Relaxed) { + let ptr = if TRACE.load(Ordering::Relaxed) { unsafe { TracingAllocator.alloc(layout) } } else { unsafe { System.alloc(layout) } + }; + if !ptr.is_null() { + count(layout.size() as isize); } + ptr } unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - if TRACE.load(Ordering::Relaxed) { + let ptr = if TRACE.load(Ordering::Relaxed) { unsafe { TracingAllocator.alloc_zeroed(layout) } } else { unsafe { System.alloc_zeroed(layout) } + }; + if !ptr.is_null() { + count(layout.size() as isize); } + ptr } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + count(-(layout.size() as isize)); if TRACE.load(Ordering::Relaxed) { unsafe { TracingAllocator.dealloc(ptr, layout) } } else { @@ -102,11 +122,15 @@ unsafe impl GlobalAlloc for Switch { } unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - if TRACE.load(Ordering::Relaxed) { + let next = if TRACE.load(Ordering::Relaxed) { unsafe { TracingAllocator.realloc(ptr, layout, new_size) } } else { unsafe { System.realloc(ptr, layout, new_size) } + }; + if !next.is_null() { + count(new_size as isize - layout.size() as isize); } + next } } @@ -326,6 +350,7 @@ fn start_resident_sampler() { std::thread::spawn(|| loop { if let Some(bytes) = resident_bytes() { RESIDENT_PEAK.fetch_max(bytes, Ordering::Relaxed); + STAGE_RSS.fetch_max(bytes, Ordering::Relaxed); } std::thread::sleep(Duration::from_millis(250)); }); @@ -391,6 +416,53 @@ fn start_tracing() { start_resident_sampler(); } +struct Stage { + name: &'static str, + started: Instant, + cpu: (f64, f64, u64), + live: isize, +} + +static STAGE: Mutex> = Mutex::new(None); + +/// Stage peaks count the whole Rust heap, including allocations made by a +/// previous stage. Counting is opt-in because atomics on every allocation +/// perturb CPU timings. C-library memory is visible only in the RSS samples. +fn stage(name: &'static str) { + if !COUNTING.load(Ordering::Relaxed) { + return; + } + let mut current = STAGE.lock().unwrap(); + let live = LIVE.load(Ordering::Relaxed); + let peak = PEAK.swap(live, Ordering::Relaxed); + let rss = resident_bytes().unwrap_or(0); + let rss_peak = STAGE_RSS.swap(rss, Ordering::Relaxed).max(rss); + let cpu = usage(); + if let Some(previous) = current.take() { + println!( + "{}", + serde_json::json!({ + "refresh_stage": previous.name, + "wall_s": previous.started.elapsed().as_secs_f64(), + "cpu_s": cpu.0 + cpu.1 - previous.cpu.0 - previous.cpu.1, + "live_bytes_at_start": previous.live, + "peak_live_bytes": peak, + "live_bytes_at_end": live, + "resident_peak_bytes": rss_peak, + "malloc_at_end": malloc_figures(), + }) + ); + } + if name != "end" { + *current = Some(Stage { + name, + started: Instant::now(), + cpu: usage(), + live, + }); + } +} + /// The probe's edit: a function appended to AFT's crate root, which most of /// the crate reads. const EDITED: &str = "crates/aft/src/lib.rs"; @@ -755,26 +827,97 @@ fn main() { Some("restart") => restart(args), Some("replay") => replay(args), Some("transition") => transition(args), - _ => panic!("usage: alloc_trace_probe overhead|burst|refresh|restart|replay|transition ..."), + _ => { + panic!("usage: alloc_trace_probe overhead|burst|refresh|restart|replay|transition ...") + } } } -/// Measures a new checkout whose revision differs from its published sibling. -/// The supplied roots stay intact; the store is private to this run. +/// Measures a revision transition using sibling admission, a refresh of an +/// already published checkout, or a cold rebuild reusing its stored records. +/// Roots must be exported source trees, not repositories with private files. +/// Their contents stay intact; the mutable checkout and store are private. fn transition(mut args: impl Iterator) { let base = std::fs::canonicalize(args.next().unwrap()).unwrap(); let target = std::fs::canonicalize(args.next().unwrap()).unwrap(); let threads = args.next().map_or(8, |threads| threads.parse().unwrap()); + let strategy = args.next().unwrap_or_else(|| "sibling".to_string()); + assert!(["sibling", "refresh", "cold"].contains(&strategy.as_str())); + let profiling = std::env::var_os("PROBE_STAGES").is_some(); + COUNTING.store(profiling, Ordering::Relaxed); + #[cfg(feature = "test-hooks")] + if profiling { + aft::data_layer::refresh_probe::set_observer(stage); + } + #[cfg(not(feature = "test-hooks"))] + assert!(!profiling, "PROBE_STAGES requires --features test-hooks"); start_tracing(); let scratch = tempfile::tempdir().unwrap(); let store = scratch.path().join("store"); let family = FamilyId::from_digest(Digest128::of(b"alloc trace transition")); let layer = Arc::new(DataLayer::new(256 << 20, threads)); - let sibling = Checkout::bind(Arc::clone(&layer), &store, family, &base).unwrap(); - phase("base_build", || format!("{:?}", sibling.build().unwrap())); - let worktree = Checkout::bind(Arc::clone(&layer), &store, family, &target).unwrap(); + let root = scratch.path().join("checkout"); + copy_tree(&base, &root); + let sibling = Checkout::bind(Arc::clone(&layer), &store, family, &root).unwrap(); + stage("base_start"); + phase("base_build", || { + let report = sibling.build().unwrap(); + stage("end"); + format!("{report:?}") + }); + let next = if strategy == "sibling" { + let next = scratch.path().join("sibling"); + copy_tree(&target, &next); + next + } else { + std::fs::remove_dir_all(&root).unwrap(); + copy_tree(&target, &root); + root + }; + stage("bind"); + let worktree = Checkout::bind(Arc::clone(&layer), &store, family, &next).unwrap(); phase("target_build", || { - with_cache_counts(&layer, || format!("{:?}", worktree.build().unwrap())) + with_cache_counts(&layer, || { + let detail = if strategy == "refresh" { + let stats = worktree.refresh_callgraph().unwrap().unwrap(); + let detail = format!( + "changed {}, rebound {}, resolved {}, replayed {}, dispatch replayed {}, recomputed {}, sites {}, touched {}; update {} ms (facts {}, bind {}, resolve {}, dispatch {}, sites {})", + stats.changed_paths, stats.rebound, stats.resolved, stats.replayed, + stats.dispatch_replayed, stats.dispatch_recomputed, stats.sites_resolved, + stats.touched.len(), stats.total_ms, stats.facts_ms, stats.bind_ms, + stats.resolve_ms, stats.dispatch_ms, stats.sites_ms, + ); + stage("publication"); + worktree.publish_changes().unwrap(); + detail + } else { + format!("{:?}", worktree.build().unwrap()) + }; + stage("end"); + detail + }) }); println!("engine parts {:?}", worktree.engine_memory_parts()); } + +/// Copies an archive's source tree without following links or copying Git's +/// metadata. The empty marker makes ignore rules behave as in a repository. +fn copy_tree(source: &Path, destination: &Path) { + std::fs::create_dir_all(destination).unwrap(); + let mut archive = Command::new("tar") + .args(["-c", "--exclude=.git", "-C"]) + .arg(source) + .arg(".") + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let status = Command::new("tar") + .arg("-x") + .arg("-C") + .arg(destination) + .stdin(archive.stdout.take().unwrap()) + .status() + .unwrap(); + assert!(status.success() && archive.wait().unwrap().success()); + std::fs::create_dir_all(destination.join(".git")).unwrap(); +} diff --git a/crates/aft/src/data_layer/callgraph.rs b/crates/aft/src/data_layer/callgraph.rs index a2523ac8f4..8dff1ae854 100644 --- a/crates/aft/src/data_layer/callgraph.rs +++ b/crates/aft/src/data_layer/callgraph.rs @@ -592,6 +592,7 @@ impl EncodedResolution { } impl Resolutions { + #[cfg(feature = "test-hooks")] pub(crate) fn profile_parts(&self) -> [usize; 5] { use aft_store::heap::HeapBytes as _; let mut seen = aft_store::heap::Seen::new(); @@ -1286,6 +1287,8 @@ pub(crate) fn resolve_checkout_into( crate::alloc_trace::install(&pool(threads)?, || { let paths = inputs.parses.keys().cloned().collect::>(); let bound = BindCache::default(); + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("binding"); // The whole-checkout lists read every parse, so they are built here, // a batch at a time: each parse is decoded once, for its kinds, and // kept until binding (which keeps none it decodes) has read it. @@ -1315,6 +1318,8 @@ pub(crate) fn resolve_checkout_into( relieve_allocator(); lap(&mut memory, "bind"); let mut resolved = Vec::with_capacity(paths.len()); + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("resolution"); for batch in paths.chunks(BATCH) { let parses = &inputs.parses; resolved.extend(resolve_files_with( @@ -1368,6 +1373,8 @@ pub(crate) fn resolve_checkout_into( // in parallel, and the sink builds their fragments in parallel too; // a whole batch decoded and built at once held about 750 MB more. let mut sites_ms = 0u128; + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("sites"); for batch in paths.chunks(BATCH) { let started = std::time::Instant::now(); let mut reached = calls diff --git a/crates/aft/src/data_layer/checkout.rs b/crates/aft/src/data_layer/checkout.rs index 98518988b1..879a82957a 100644 --- a/crates/aft/src/data_layer/checkout.rs +++ b/crates/aft/src/data_layer/checkout.rs @@ -1533,6 +1533,8 @@ impl Checkout { stored: self.stored_records()?, ..BuildSink::default() }; + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("extraction"); let scanned = scan(&self.root, &self.layer.policy, None, &sink) .map_err(|error| DataError::Scan(error.to_string()))?; let mut collected = sink.finish(); @@ -1650,6 +1652,8 @@ impl Checkout { report.resolve_ms = elapsed_ms(started); checkpoint()?; + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("publication"); let started = Instant::now(); #[cfg(test)] wait_while_publication_held_for_test(&self.root); @@ -1865,6 +1869,8 @@ impl Checkout { log::debug!("sibling callgraph update: {update:?}"); let resolve_ms = elapsed_ms(started); let started = Instant::now(); + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("publication"); let delta = self.publish_changes_as(true)?; let publish_ms = elapsed_ms(started); let segment_bytes = self @@ -2645,6 +2651,8 @@ impl Checkout { let mut stats = None; if !changed.is_empty() { note_source_work(&self.root, |work| work.resolutions += 1); + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("extraction"); // The changed files are extracted here, all at once on the // engine's threads, rather than each when its edit was recorded. let extracting = changed @@ -2659,6 +2667,8 @@ impl Checkout { change.content.get(); }); }); + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("planning"); let next = next_inputs(state, &engine.live.inputs, changes, &changed); let changed_names = changed .iter() @@ -2677,6 +2687,9 @@ impl Checkout { // The caller drops a partly updated engine after either error, // so the next refresh starts again from the publication. let update = update.map_err(|error| DataError::Resolve(error.to_string()))?; + log::debug!("callgraph refresh work: changed={} rebound={} replayed={} resolved={} dispatch_replayed={} dispatch_recomputed={} sites={} touched={}", update.changed_paths, update.rebound, update.replayed, update.resolved, update.dispatch_replayed, update.dispatch_recomputed, update.sites_resolved, update.touched.len()); + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("slices"); // Only the files this update touched get new slices; every file // touched by an earlier update keeps the slice it got then. Each // is laid over the publication encoded, as a publication stores diff --git a/crates/aft/src/data_layer/incremental.rs b/crates/aft/src/data_layer/incremental.rs index 1226095723..a612f3b0ec 100644 --- a/crates/aft/src/data_layer/incremental.rs +++ b/crates/aft/src/data_layer/incremental.rs @@ -613,6 +613,8 @@ impl LiveCheckout { let pool = Arc::clone(&self.pool); let mut stats = crate::alloc_trace::install(&pool, || { let stats = self.update_in_pool(next, changed)?; + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("encode"); // Every resolution the update made or changed is encoded, so // that it can be let go between updates like a stored one. let parses = &self.inputs.parses; @@ -714,7 +716,9 @@ impl LiveCheckout { ..UpdateStats::default() }; let old = std::mem::replace(&mut self.inputs, next); + #[cfg(feature = "test-hooks")] let mut profile_clock = Instant::now(); + #[cfg(feature = "test-hooks")] self.profile_refresh("start", &mut profile_clock); let inputs = &self.inputs; let membership = changed @@ -856,6 +860,8 @@ impl LiveCheckout { // read is asked again; a file any of whose answers differ is bound, // resolved or traced again. This needs no rule about which change can // affect which read. + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("facts"); let started = Instant::now(); let stale = if membership.is_empty() && links.is_empty() && configs.is_empty() { StaleAnswers::default() @@ -866,10 +872,13 @@ impl LiveCheckout { stats.answers_checked = stale.checked; stats.answers_changed = stale.changed; stats.facts_ms = started.elapsed().as_secs_f64() * 1000.0; + #[cfg(feature = "test-hooks")] self.profile_refresh("facts", &mut profile_clock); // Pass 1. A binding depends on the file's own parse, on which paths // exist, and on configuration it read. + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("binding"); let started = Instant::now(); let mut binding_keys = membership.clone(); binding_keys.extend(changed_packages.iter().cloned()); @@ -989,6 +998,7 @@ impl LiveCheckout { self.bound.set(path, bound); } stats.bind_ms = started.elapsed().as_secs_f64() * 1000.0; + #[cfg(feature = "test-hooks")] self.profile_refresh("bind", &mut profile_clock); // What a recorded answer can depend on that changed. @@ -1011,6 +1021,8 @@ impl LiveCheckout { changed_keys.extend(changed_crates); // Pass 2. + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("replay"); let started = Instant::now(); let facts = Rc::new(ViewBindingFacts::typed(Rc::new(CheckoutFacts { inputs }))); let (index, files) = resolver_index(inputs, &self.bound, &facts); @@ -1096,7 +1108,10 @@ impl LiveCheckout { } drop(answers); drop(trace_answers); + #[cfg(feature = "test-hooks")] self.profile_refresh("replay", &mut profile_clock); + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("resolution"); // Files kept whose binding was redone carry their resolution over. for path in &rebind { if resolve.contains(path) { @@ -1137,12 +1152,15 @@ impl LiveCheckout { self.resolved.insert(path, file); } stats.resolve_ms = started.elapsed().as_secs_f64() * 1000.0; + #[cfg(feature = "test-hooks")] self.profile_refresh("resolve", &mut profile_clock); // Pass 3, for files whose rows were kept: each trace was replayed // with its file's surface answers above. Resolving files again // changes no surface a trace reads, since every file whose binding // could change was bound again in pass 1. + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("dispatch"); let started = Instant::now(); let mut recompute = Vec::new(); // Stale answers were asked before this update removed files; a @@ -1180,9 +1198,12 @@ impl LiveCheckout { } drop(index); drop(facts); + #[cfg(feature = "test-hooks")] self.profile_refresh("dispatch", &mut profile_clock); // Pass 4. + #[cfg(feature = "test-hooks")] + super::refresh_probe::stage("sites"); let started = Instant::now(); let mut site_keys = BTreeSet::new(); let mut sites = contents.clone(); @@ -1219,6 +1240,7 @@ impl LiveCheckout { } stats.sites_resolved = paths.len(); stats.sites_ms = started.elapsed().as_secs_f64() * 1000.0; + #[cfg(feature = "test-hooks")] self.profile_refresh("sites", &mut profile_clock); stats.touched = rebind .into_iter() @@ -1230,6 +1252,7 @@ impl LiveCheckout { Ok(stats) } + #[cfg(feature = "test-hooks")] fn profile_refresh(&self, stage: &str, clock: &mut Instant) { if std::env::var_os("AFT_REFRESH_PROFILE").is_none() { return; diff --git a/crates/aft/src/data_layer/mod.rs b/crates/aft/src/data_layer/mod.rs index 9e7d7cbd6a..66aa85b494 100644 --- a/crates/aft/src/data_layer/mod.rs +++ b/crates/aft/src/data_layer/mod.rs @@ -27,6 +27,8 @@ pub mod parent_session; #[cfg(test)] mod publication_tests; pub(crate) mod query; +#[cfg(feature = "test-hooks")] +pub mod refresh_probe; pub mod reuse; pub(crate) mod runtime; #[cfg(test)] diff --git a/crates/aft/src/data_layer/refresh_probe.rs b/crates/aft/src/data_layer/refresh_probe.rs new file mode 100644 index 0000000000..43c6856172 --- /dev/null +++ b/crates/aft/src/data_layer/refresh_probe.rs @@ -0,0 +1,16 @@ +//! Stage boundaries for the allocation probe. Absent from production builds. + +use std::sync::OnceLock; + +static OBSERVER: OnceLock = OnceLock::new(); + +/// Installs the process-wide measurement callback before a probe starts. +pub fn set_observer(observer: fn(&'static str)) { + let _ = OBSERVER.set(observer); +} + +pub(crate) fn stage(name: &'static str) { + if let Some(observer) = OBSERVER.get() { + observer(name); + } +} From bc9ca4ea48b72968e78386c673508d0d6cff363a Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:15:11 +0100 Subject: [PATCH 004/163] mason: bound refresh invalidations with a single-flight cold fallback Admit each dependency pass before doing its invalidated work, counting the union since publication. Above max(1024, ceil(parsed/4)), drop the incremental engine before rebuilding from stored content. Reserve cold builds while queued and running; retain later watcher changes for post-publication catch-up. The existing concurrent-build test intentionally changes contract: refresh now coalesces with a cold build instead of resolving over the publication that the build will replace. It still asserts that late changes stay pending and resolve correctly after publication. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 8d752ed597fb2dc83773b6007f803f90bad9d870) (cherry picked from commit 3e8f2a640219578fc98eb6a8b15c3cfa34a3a247) --- crates/aft/examples/alloc_trace_probe.rs | 20 +- crates/aft/src/data_layer/checkout.rs | 339 ++++++++++++++++++-- crates/aft/src/data_layer/checkout_gate.rs | 21 ++ crates/aft/src/data_layer/incremental.rs | 56 +++- crates/aft/src/data_layer/mod.rs | 1 + crates/aft/src/data_layer/refresh_budget.rs | 47 +++ crates/aft/src/data_layer/runtime.rs | 11 +- 7 files changed, 454 insertions(+), 41 deletions(-) create mode 100644 crates/aft/src/data_layer/refresh_budget.rs diff --git a/crates/aft/examples/alloc_trace_probe.rs b/crates/aft/examples/alloc_trace_probe.rs index cddf423929..99c8d9a9a7 100644 --- a/crates/aft/examples/alloc_trace_probe.rs +++ b/crates/aft/examples/alloc_trace_probe.rs @@ -725,6 +725,7 @@ fn replay(mut args: impl Iterator) { let mut reused = (0usize, 0.0f64, 0.0f64); let mut walls = Vec::new(); let mut publications = 0usize; + let mut cold_builds = 0usize; phase("replay", || { let mut since = 0usize; for (index, edit) in edits.iter().enumerate() { @@ -735,6 +736,7 @@ fn replay(mut args: impl Iterator) { let cpu = cpu_seconds(); let wall = Instant::now(); let stats = checkout.refresh_callgraph().unwrap(); + cold_builds += usize::from(stats.as_ref().is_some_and(|stats| stats.cold_build)); let wall = wall.elapsed().as_secs_f64(); let cpu = cpu_seconds() - cpu; if stats.is_some() { @@ -767,7 +769,7 @@ fn replay(mut args: impl Iterator) { .unwrap_or(0.0) }; format!( - "{} edits, {publications} publications; refreshes that started an engine: {} \ + "{} edits, {publications} publications, {cold_builds} cold switches; refreshes that started an engine: {} \ ({:.1} s wall, {:.1} s CPU); that reused one: {} ({:.1} s wall, {:.1} s CPU); \ all: {:.1} s wall, {:.1} s CPU, per refresh p50 {:.0} ms, p90 {:.0} ms, max {:.0} ms", edits.len(), @@ -898,6 +900,22 @@ fn transition(mut args: impl Iterator) { }) }); println!("engine parts {:?}", worktree.engine_memory_parts()); + #[cfg(feature = "test-hooks")] + if std::env::var_os("PROBE_COMPARE").is_some() { + let cold = Checkout::bind( + Arc::clone(&layer), + &scratch.path().join("cold-store"), + family, + &next, + ) + .unwrap(); + cold.build().unwrap(); + let compared = aft::data_layer::checkout_gate::compare_engines(&worktree, &cold).unwrap(); + println!( + "forced-switch comparison {}", + serde_json::to_string(&compared).unwrap() + ); + } } /// Copies an archive's source tree without following links or copying Git's diff --git a/crates/aft/src/data_layer/checkout.rs b/crates/aft/src/data_layer/checkout.rs index 879a82957a..7e661d8cac 100644 --- a/crates/aft/src/data_layer/checkout.rs +++ b/crates/aft/src/data_layer/checkout.rs @@ -34,9 +34,10 @@ use aft_store::text::{ use super::callgraph::{CheckoutInputs, EncodedResolution, FileResolution}; use super::content::{blob_from_record, extract, language_for, EXTRACTOR_VERSION}; use super::graph_store::{Held, SectionsWriter, StoredSlices}; -use super::incremental::{LiveCheckout, UpdateStats}; +use super::incremental::{LiveCheckout, UpdateError, UpdateStats}; use super::overlay::{CacheKey, OverlaySlices, PreparedSlice}; use super::query::{SliceReader, SliceSource}; +use super::refresh_budget::{RefreshBudget, RefreshLimit}; use super::slice::FileSlice; use crate::callgraph_store::join::{CallgraphBlob, ParseBlob}; @@ -79,6 +80,8 @@ pub(crate) struct SourceWork { /// Passes over the files: a scan reads every file whose stat does not /// prove its recorded content, and a build's scan parses what it reads. pub(crate) scans: usize, + /// Whole call-graph cold builds, excluding text-only scans and sibling admission. + pub(crate) cold_builds: usize, /// Call-graph resolutions: a build's, or a refresh's update. pub(crate) resolutions: usize, /// The most stored parses, and separately resolutions, a refresh's @@ -372,6 +375,29 @@ struct GraphEngine { measure_after_publication: bool, } +enum EngineUpdate { + Applied( + Option, + Option<(OverlaySlices>, Refreshed)>, + ), + Cold(RefreshLimit), +} + +/// A checkout's single queued or running build. Dropping a queued task also +/// releases its reservation, so another reader can request the build again. +pub(crate) struct BuildReservation { + live: Arc>, +} + +impl Drop for BuildReservation { + fn drop(&mut self) { + self.live + .write() + .expect("no panics while holding it") + .building = false; + } +} + impl GraphEngine { /// This engine moved onto `to`, the publication of what it resolved: /// the engine a start from `to` gives, keeping every file it bound and @@ -1451,11 +1477,42 @@ impl Checkout { } fn build_as(&self, measure: bool) -> Result { + let Some(reservation) = self.reserve_build() else { + return Ok(BuildReport { + superseded: true, + ..BuildReport::default() + }); + }; + self.build_reserved(measure, reservation) + } + + pub(crate) fn reserve_build(&self) -> Option { + let mut live = self.write_live(); + if live.building { + return None; + } + live.building = true; + Some(BuildReservation { + live: Arc::clone(&self.live), + }) + } + + pub(crate) fn is_building(&self) -> bool { + self.live + .read() + .expect("no panics while holding it") + .building + } + + pub(crate) fn build_reserved( + &self, + measure: bool, + reservation: BuildReservation, + ) -> Result { let _alloc_label = crate::alloc_trace::scope(crate::alloc_trace::Label::INDEX_BUILD, Some(&self.root)); - self.write_live().building = true; let result = self.build_inner(measure); - self.write_live().building = false; + drop(reservation); self.note_outcome("build", &result); result } @@ -1512,6 +1569,7 @@ impl Checkout { checkpoint()?; // A change recorded before the scan starts is in what the scan // reads; one recorded later may not be. + note_source_work(&self.root, |work| work.cold_builds += 1); let covered = self .live .read() @@ -1865,7 +1923,7 @@ impl Checkout { let text_ready_ms = elapsed_ms(build_started); checkpoint()?; let started = Instant::now(); - let update = self.refresh_callgraph()?; + let update = self.refresh_callgraph_as(true)?; log::debug!("sibling callgraph update: {update:?}"); let resolve_ms = elapsed_ms(started); let started = Instant::now(); @@ -2480,12 +2538,16 @@ impl Checkout { /// When a stored record or fragment cannot be read, or resolution /// fails. The pending paths stay pending then. pub fn refresh_callgraph(&self) -> Result, DataError> { + self.refresh_callgraph_as(false) + } + + fn refresh_callgraph_as(&self, by_build: bool) -> Result, DataError> { let _alloc_label = crate::alloc_trace::scope( crate::alloc_trace::Label::CALLGRAPH_REFRESH, Some(&self.root), ); checkpoint()?; - let result = self.refresh_inner(); + let result = self.refresh_inner(by_build); self.touch_memory(); self.note_outcome("refresh", &result); result @@ -2503,9 +2565,12 @@ impl Checkout { .fetch_add(1, std::sync::atomic::Ordering::AcqRel); } - fn refresh_inner(&self) -> Result, DataError> { - let (published, state, changes, covered) = { + fn refresh_inner(&self, by_build: bool) -> Result, DataError> { + let (published, state, changes, covered, invalidated) = { let live = self.live.read().expect("no panics while holding it"); + if live.building && !by_build { + return Ok(None); + } let Some(published) = live.published.clone() else { return Ok(None); }; @@ -2513,7 +2578,24 @@ impl Checkout { .state .clone() .unwrap_or_else(|| Arc::new(published.state.clone())); - (published, state, live.changes.clone(), live.sequence) + let invalidated = live + .refreshed + .as_ref() + .map_or_else(BTreeSet::new, |refreshed| { + refreshed + .replaced + .keys() + .chain(&refreshed.removed) + .cloned() + .collect() + }); + ( + published, + state, + live.changes.clone(), + live.sequence, + invalidated, + ) }; // Without a call graph there is nothing to update; the build that // makes one covers every change recorded before its scan. @@ -2529,15 +2611,55 @@ impl Checkout { *slot = None; slot }); + if self.is_building() && !by_build { + return Ok(None); + } + let parsed = slot.as_ref().map_or_else( + || published.graph.parsed_files().count(), + |engine| engine.live.inputs.parses.len(), + ); + let mut budget = RefreshBudget::new(parsed, invalidated); // A damaged stored record or fragment can stop the update where it // expects one (every file the engine lists has a parse and a // resolution). Such a stop fails this refresh as damage, and the // engine, whose state it may have left half-updated, starts again. let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - self.update_engine(&mut slot, &published, &state, &changes) + self.update_engine(&mut slot, &published, &state, &changes, &mut budget) })); let (stats, served) = match outcome { - Ok(Ok(outcome)) => outcome, + Ok(Ok(EngineUpdate::Applied(stats, served))) => (stats, served), + Ok(Ok(EngineUpdate::Cold(limit))) => { + // No incremental engine or its decoded state overlaps the cold + // builder. Watcher events keep recording over the publication. + *slot = None; + let reservation = if by_build { None } else { self.reserve_build() }; + drop(slot); + drop(changes); + drop(state); + drop(published); + if !by_build && reservation.is_none() { + return Ok(None); + } + crate::slog_info!( + "data layer cold refresh: root={} invalidated={} threshold={} parsed={}", + self.root.display(), + limit.invalidated, + limit.threshold, + limit.parsed + ); + let started = Instant::now(); + if let Some(reservation) = reservation { + self.build_reserved(false, reservation)?; + } else { + // Sibling admission already owns the build reservation. + self.build_inner(false)?; + } + return Ok(Some(UpdateStats { + cold_build: true, + total_ms: elapsed_ms(started), + ..UpdateStats::default() + })); + } Ok(Err(error)) => { *slot = None; return Err(error); @@ -2625,13 +2747,20 @@ impl Checkout { published: &Arc, state: &Arc, changes: &BTreeMap, Change>, - ) -> Result< - ( - Option, - Option<(OverlaySlices>, Refreshed)>, - ), - DataError, - > { + budget: &mut RefreshBudget, + ) -> Result { + // This test needs neither extraction nor an engine start. A single + // watcher batch can already be larger than incremental admission. + if let Err(limit) = budget.check(changes.len()) { + return Ok(EngineUpdate::Cold(limit)); + } + let changed_members = changes + .keys() + .filter_map(|path| std::str::from_utf8(path).ok().map(str::to_owned)) + .collect::>(); + if let Err(limit) = budget.include(&changed_members) { + return Ok(EngineUpdate::Cold(limit)); + } if slot .as_ref() .is_some_and(|engine| !Arc::ptr_eq(&engine.published, &published)) @@ -2670,11 +2799,16 @@ impl Checkout { #[cfg(feature = "test-hooks")] super::refresh_probe::stage("planning"); let next = next_inputs(state, &engine.live.inputs, changes, &changed); + if let Err(limit) = budget.set_parsed(next.parses.len()) { + return Ok(EngineUpdate::Cold(limit)); + } let changed_names = changed .iter() .filter_map(|path| std::str::from_utf8(path).ok().map(str::to_string)) .collect::>(); - let update = engine.live.update(next, &changed_names); + let update = engine + .live + .update_with_budget(next, &changed_names, Some(budget)); // The engine reads stored parses and resolutions as it goes; a // read that failed made it work from an empty one. let failures = published.graph.failures(); @@ -2686,7 +2820,13 @@ impl Checkout { } // The caller drops a partly updated engine after either error, // so the next refresh starts again from the publication. - let update = update.map_err(|error| DataError::Resolve(error.to_string()))?; + let update = match update { + Ok(update) => update, + Err(UpdateError::Limit(limit)) => return Ok(EngineUpdate::Cold(limit)), + Err(UpdateError::Resolve(error)) => { + return Err(DataError::Resolve(error.to_string())) + } + }; log::debug!("callgraph refresh work: changed={} rebound={} replayed={} resolved={} dispatch_replayed={} dispatch_recomputed={} sites={} touched={}", update.changed_paths, update.rebound, update.replayed, update.resolved, update.dispatch_replayed, update.dispatch_recomputed, update.sites_resolved, update.touched.len()); #[cfg(feature = "test-hooks")] super::refresh_probe::stage("slices"); @@ -2779,7 +2919,7 @@ impl Checkout { let served = stats .is_some() .then(|| (engine.overlay.clone(), engine.refreshed.clone())); - Ok((stats, served)) + Ok(EngineUpdate::Applied(stats, served)) } /// Publishes what changed since the publication in a segment of its @@ -6251,7 +6391,13 @@ pub fn other() { callee(); } ) }; assert!(matches!( - checkout.update_engine(&mut None, &published, &state, &changes), + checkout.update_engine( + &mut None, + &published, + &state, + &changes, + &mut RefreshBudget::new(published.graph.parsed_files().count(), BTreeSet::new()) + ), Err(DataError::Damaged(_)) )); for attempt in 0..2 { @@ -7116,14 +7262,11 @@ pub fn other() { callee(); } ); } - /// A change recorded while a build runs, after its scan, stays laid over - /// the build's publication. A refresh that resolved it during the build - /// laid its answer over the old publication's call graph, which the - /// build's replaces, so the change is pending again until a refresh - /// resolves it over the new one. The call graph used to report ready - /// then and answer without the change. + /// A change after a build's scan stays pending over its publication. + /// Refreshes now coalesce with that build instead of resolving against + /// the old publication and discarding those results when the build ends. #[test] - fn a_change_resolved_while_a_build_ran_is_resolved_again_over_its_publication() { + fn a_change_recorded_while_a_build_runs_resolves_over_its_publication() { let root = tempfile::tempdir().unwrap(); let store = tempfile::tempdir().unwrap(); demo(root.path()); @@ -7152,12 +7295,8 @@ pub fn other() { callee(); } checkout .record_paths(&[b"src/extra.rs".to_vec(), b"src/lib.rs".to_vec()]) .unwrap(); - checkout.refresh_callgraph().unwrap(); - let view = checkout.view().unwrap(); - assert_eq!(view.status().callgraph, PlaneStatus::Ready); - let resolved = callers_of(&view, "src/util.rs", "callee"); - assert!(resolved.contains("src/extra.rs"), "{resolved}"); - drop(view); + assert!(checkout.refresh_callgraph().unwrap().is_none()); + assert_eq!(checkout.status().callgraph_pending, 2); drop(hold); building.join().unwrap().unwrap(); @@ -7171,7 +7310,7 @@ pub fn other() { callee(); } checkout.refresh_callgraph().unwrap(); assert_eq!(checkout.status().callgraph, PlaneStatus::Ready); let callers = callers_of(&checkout.view().unwrap(), "src/util.rs", "callee"); - assert_eq!(callers, resolved); + assert!(callers.contains("src/extra.rs"), "{callers}"); let cold_store = tempfile::tempdir().unwrap(); let cold = Checkout::bind(layer, cold_store.path(), family(), root.path()).unwrap(); cold.build().unwrap(); @@ -7596,6 +7735,136 @@ pub fn other() { callee(); } const CHAIN: usize = 8; + #[test] + fn bounded_refresh_accumulates_invalidations_across_two_passes() { + let store = tempfile::tempdir().unwrap(); + let layer = Arc::new(DataLayer::new(8 << 20, 2)); + let mut files = BTreeMap::new(); + for group in ["a", "b"] { + files.insert( + format!("{group}.ts"), + format!("export function helper_{group}() {{}}\n"), + ); + for i in 0..550 { + files.insert(format!("{group}{i}.ts"), format!( + "import {{ helper_{group} }} from './{group}';\nexport function f_{group}{i}() {{ helper_{group}(); }}\n" + )); + } + } + let worktree = Worktree::new(&layer, store.path(), files); + let root = worktree.dir.path(); + let checkout = &worktree.checkout; + let before = source_work_for_test(root).cold_builds; + let edit = |group: &str| { + let path = format!("{group}.ts"); + write( + root, + &path, + &format!("export function renamed_{group}() {{}}\n"), + ); + checkout.record_paths(&[path.into_bytes()]).unwrap(); + checkout.refresh_callgraph().unwrap().unwrap() + }; + let first = edit("a"); + assert!(!first.cold_build); + assert!((500..1024).contains(&first.touched.len()), "{first:?}"); + assert_eq!(source_work_for_test(root).cold_builds, before); + let second = edit("b"); + assert!( + second.cold_build, + "two disjoint sub-threshold passes exceed the union limit" + ); + assert_eq!(source_work_for_test(root).cold_builds, before + 1); + assert_eq!(checkout.status().callgraph, PlaneStatus::Ready); + assert!(checkout.refresh_callgraph().unwrap().is_none()); + assert_eq!(source_work_for_test(root).cold_builds, before + 1); + let cold_store = tempfile::tempdir().unwrap(); + let cold = Checkout::bind(Arc::clone(&layer), cold_store.path(), family(), root).unwrap(); + cold.build().unwrap(); + let compared = super::super::checkout_gate::compare_engines(checkout, &cold).unwrap(); + assert_eq!(compared.files, 1102); + } + + fn record_many_notes(checkout: &Checkout, root: &Path, generation: usize) { + let paths = (0..1025) + .map(|i| { + let path = format!("notes_{generation}_{i}.txt"); + write(root, &path, "changed while the checkout is being rebased\n"); + path.into_bytes() + }) + .collect::>(); + checkout.record_paths(&paths).unwrap(); + } + + fn bounded_refresh_catch_up(broad: bool) { + let root = tempfile::tempdir().unwrap(); + let store = tempfile::tempdir().unwrap(); + demo(root.path()); + let layer = Arc::new(DataLayer::new(8 << 20, 2)); + let checkout = + Arc::new(Checkout::bind(layer, store.path(), family(), root.path()).unwrap()); + checkout.build().unwrap(); + let before = source_work_for_test(root.path()).cold_builds; + record_many_notes(&checkout, root.path(), 0); + let hold = hold_build_publication_for_test(root.path()); + let refreshing = { + let checkout = Arc::clone(&checkout); + std::thread::spawn(move || checkout.refresh_callgraph().unwrap().unwrap()) + }; + hold.wait_until_held(); + if broad { + record_many_notes(&checkout, root.path(), 1); + } else { + write(root.path(), "src/extra.rs", "pub fn late() {}\n"); + checkout.record_paths(&[b"src/extra.rs".to_vec()]).unwrap(); + } + assert!(checkout.refresh_callgraph().unwrap().is_none()); + assert!( + checkout.build().unwrap().superseded, + "a running cold build absorbs another request" + ); + assert_eq!(source_work_for_test(root.path()).cold_builds, before + 1); + drop(hold); + assert!(refreshing.join().unwrap().cold_build); + assert!(checkout.status().callgraph_pending > 0); + let caught_up = checkout.refresh_callgraph().unwrap().unwrap(); + assert_eq!(caught_up.cold_build, broad); + assert_eq!( + source_work_for_test(root.path()).cold_builds, + before + 1 + usize::from(broad) + ); + assert_eq!(checkout.status().callgraph, PlaneStatus::Ready); + } + + #[test] + fn bounded_refresh_catches_up_small_events_during_a_cold_build_incrementally() { + bounded_refresh_catch_up(false); + } + + #[test] + fn bounded_refresh_runs_one_more_cold_build_for_a_broad_preserved_batch() { + bounded_refresh_catch_up(true); + } + + #[test] + fn bounded_refresh_coalesces_with_a_queued_cold_build() { + let root = tempfile::tempdir().unwrap(); + let store = tempfile::tempdir().unwrap(); + demo(root.path()); + let layer = Arc::new(DataLayer::new(8 << 20, 2)); + let checkout = Checkout::bind(layer, store.path(), family(), root.path()).unwrap(); + checkout.build().unwrap(); + record_many_notes(&checkout, root.path(), 0); + let before = source_work_for_test(root.path()).cold_builds; + let reservation = checkout.reserve_build().unwrap(); + assert!(checkout.refresh_callgraph().unwrap().is_none()); + assert!(checkout.reserve_build().is_none()); + assert_eq!(source_work_for_test(root.path()).cold_builds, before); + checkout.build_reserved(false, reservation).unwrap(); + assert_eq!(source_work_for_test(root.path()).cold_builds, before + 1); + assert_eq!(checkout.status().callgraph, PlaneStatus::Ready); + } + /// Worktrees of one repository keep editing while a main checkout takes /// in their changes, as one merging its workers' branches does, and a /// new worktree is made from main every few rounds; older worktrees go diff --git a/crates/aft/src/data_layer/checkout_gate.rs b/crates/aft/src/data_layer/checkout_gate.rs index d1e4c58f72..5ad800e905 100644 --- a/crates/aft/src/data_layer/checkout_gate.rs +++ b/crates/aft/src/data_layer/checkout_gate.rs @@ -162,6 +162,27 @@ impl EngineComparison { } } +/// Compares every engine input, answer, recording and encoded fragment after a +/// probe has forced a refresh to switch to cold. Unlike a query sample, this +/// fails when any file differs or when no file was compared. +#[cfg(any(test, feature = "test-hooks"))] +pub fn compare_engines(checkout: &Checkout, cold: &Checkout) -> Result { + let mut examples = Vec::new(); + let compared = cold + .inspect_engine(|cold| { + checkout.inspect_engine(|live| compare_engine(live, cold, &mut examples)) + }) + .map_err(|error| error.to_string())? + .ok_or("the cold build published no call graph")?; + let comparison = compared + .map_err(|error| error.to_string())? + .ok_or("the checkout published no call graph")?; + if comparison.failed() { + return Err(format!("{comparison:?}: {examples:?}")); + } + Ok(comparison) +} + /// A folder under the system's temporary folder, removed when dropped. struct Scratch(PathBuf); diff --git a/crates/aft/src/data_layer/incremental.rs b/crates/aft/src/data_layer/incremental.rs index a612f3b0ec..1cc47e1bcd 100644 --- a/crates/aft/src/data_layer/incremental.rs +++ b/crates/aft/src/data_layer/incremental.rs @@ -48,11 +48,14 @@ use super::callgraph::{ CheckoutInputs, FileResolution, MethodCallInputs, Resolutions, }; use super::graph_store::StoredSlices; +use super::refresh_budget::{RefreshBudget, RefreshLimit}; use super::subjects::{affected_by, subjects, Subject}; /// What one update did, for measurement. #[derive(Clone, Debug, Default)] pub struct UpdateStats { + /// A broad incremental update was replaced by a stored-record cold build. + pub cold_build: bool, pub changed_paths: usize, /// Recorded checkout reads asked again, and how many got a different /// answer. @@ -84,6 +87,27 @@ pub struct UpdateStats { pub touched: BTreeSet, } +pub(crate) enum UpdateError { + Resolve(ManifestJoinError), + Limit(RefreshLimit), +} + +impl From for UpdateError { + fn from(error: ManifestJoinError) -> Self { + Self::Resolve(error) + } +} + +fn admit<'a>( + budget: &mut Option<&mut RefreshBudget>, + files: impl IntoIterator, +) -> Result<(), UpdateError> { + if let Some(budget) = budget.as_deref_mut() { + budget.include(files).map_err(UpdateError::Limit)?; + } + Ok(()) +} + /// Which files recorded each input key, for one kind of recording: the /// reverse inputs a change is looked up in. /// @@ -609,17 +633,34 @@ impl LiveCheckout { next: CheckoutInputs, changed: &BTreeSet, ) -> Result { + self.update_with_budget(next, changed, None) + .map_err(|error| match error { + UpdateError::Resolve(error) => error, + UpdateError::Limit(_) => { + unreachable!("an unbounded update cannot hit a refresh limit") + } + }) + } + + /// A limit aborts before the newly invalidated pass runs. The caller must + /// drop this partly updated engine before rebuilding the checkout. + pub(crate) fn update_with_budget( + &mut self, + next: CheckoutInputs, + changed: &BTreeSet, + budget: Option<&mut RefreshBudget>, + ) -> Result { let started = Instant::now(); let pool = Arc::clone(&self.pool); let mut stats = crate::alloc_trace::install(&pool, || { - let stats = self.update_in_pool(next, changed)?; + let stats = self.update_in_pool(next, changed, budget)?; #[cfg(feature = "test-hooks")] super::refresh_probe::stage("encode"); // Every resolution the update made or changed is encoded, so // that it can be let go between updates like a stored one. let parses = &self.inputs.parses; self.resolved.encode_changed(|path| parses.key(path)); - Ok::<_, ManifestJoinError>(stats) + Ok::<_, UpdateError>(stats) })?; stats.total_ms = started.elapsed().as_secs_f64() * 1000.0; Ok(stats) @@ -710,7 +751,9 @@ impl LiveCheckout { &mut self, next: CheckoutInputs, changed: &BTreeSet, - ) -> Result { + mut budget: Option<&mut RefreshBudget>, + ) -> Result { + admit(&mut budget, changed)?; let mut stats = UpdateStats { changed_paths: changed.len(), ..UpdateStats::default() @@ -905,6 +948,7 @@ impl LiveCheckout { rebind.extend(contents.iter().cloned()); rebind.extend(stale.binding.iter().cloned()); rebind.retain(|path| inputs.parses.contains_key(path)); + admit(&mut budget, rebind.iter().chain(&removed_parses))?; let rebind = rebind.into_iter().collect::>(); let rebind_refs = rebind.iter().collect::>(); let new_bound = bind_all(inputs, &rebind_refs)?; @@ -1036,6 +1080,7 @@ impl LiveCheckout { resolve.insert(path.clone()); } } + admit(&mut budget, &resolve)?; // Resolving a Rust file reads the module tree, which the resolver // builds by binding every module-declaring file; bind those in // parallel first (see `BindCache::bind_module_declaring`). @@ -1108,6 +1153,7 @@ impl LiveCheckout { } drop(answers); drop(trace_answers); + admit(&mut budget, &resolve)?; #[cfg(feature = "test-hooks")] self.profile_refresh("replay", &mut profile_clock); #[cfg(feature = "test-hooks")] @@ -1174,6 +1220,7 @@ impl LiveCheckout { } } recompute.extend(traces_changed); + admit(&mut budget, &recompute)?; for path in &recompute { let (_, parse) = &inputs.parses[path.as_str()]; let dispatch = dispatch_targets(path, parse, &index, &facts)?; @@ -1194,7 +1241,7 @@ impl LiveCheckout { // A file bound on first use inside passes 2 and 3 cannot return its // error there. if let Some(error) = files.take_error() { - return Err(error); + return Err(error.into()); } drop(index); drop(facts); @@ -1225,6 +1272,7 @@ impl LiveCheckout { } sites.extend(self.reverse.sites_of(&site_keys)); sites.retain(|path| inputs.parses.contains_key(path)); + admit(&mut budget, &sites)?; let paths = sites.iter().collect::>(); for (path, kept, dependencies) in self.calls.resolve_sites(inputs, &self.resolved, &paths)? diff --git a/crates/aft/src/data_layer/mod.rs b/crates/aft/src/data_layer/mod.rs index 66aa85b494..2624d3c783 100644 --- a/crates/aft/src/data_layer/mod.rs +++ b/crates/aft/src/data_layer/mod.rs @@ -27,6 +27,7 @@ pub mod parent_session; #[cfg(test)] mod publication_tests; pub(crate) mod query; +mod refresh_budget; #[cfg(feature = "test-hooks")] pub mod refresh_probe; pub mod reuse; diff --git a/crates/aft/src/data_layer/refresh_budget.rs b/crates/aft/src/data_layer/refresh_budget.rs new file mode 100644 index 0000000000..cd2d8cef13 --- /dev/null +++ b/crates/aft/src/data_layer/refresh_budget.rs @@ -0,0 +1,47 @@ +//! Admission by the files an incremental engine has invalidated since publication. + +use std::collections::BTreeSet; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct RefreshLimit { + pub(crate) invalidated: usize, + pub(crate) threshold: usize, + pub(crate) parsed: usize, +} + +pub(crate) struct RefreshBudget { + files: BTreeSet, + parsed: usize, +} + +impl RefreshBudget { + pub(crate) fn new(parsed: usize, files: BTreeSet) -> Self { + Self { files, parsed } + } + + pub(crate) fn set_parsed(&mut self, parsed: usize) -> Result<(), RefreshLimit> { + self.parsed = parsed; + self.check(self.files.len()) + } + + pub(crate) fn check(&self, count: usize) -> Result<(), RefreshLimit> { + let threshold = self.parsed.div_ceil(4).max(1024); + if count > threshold { + Err(RefreshLimit { + invalidated: count, + threshold, + parsed: self.parsed, + }) + } else { + Ok(()) + } + } + + pub(crate) fn include<'a>( + &mut self, + files: impl IntoIterator, + ) -> Result<(), RefreshLimit> { + self.files.extend(files.into_iter().cloned()); + self.check(self.files.len()) + } +} diff --git a/crates/aft/src/data_layer/runtime.rs b/crates/aft/src/data_layer/runtime.rs index 07ca08a509..371aaa78f9 100644 --- a/crates/aft/src/data_layer/runtime.rs +++ b/crates/aft/src/data_layer/runtime.rs @@ -416,6 +416,9 @@ impl Work { return false; } let flag = ClearOnDrop(Arc::clone(&self.building)); + let Some(reservation) = self.checkout.reserve_build() else { + return; + }; let work = Arc::clone(self); self.runner.run(BUILD_REQUEST, Pool::Build, move || { #[cfg(test)] @@ -426,7 +429,7 @@ impl Work { { std::thread::sleep(Duration::from_millis(5)); } - let result = work.checkout.build(); + let result = work.checkout.build_reserved(false, reservation); drop(flag); work.checkout.notify_readiness_change(); if !matches!(result, Err(DataError::Cancelled)) { @@ -515,6 +518,9 @@ impl Work { let mut requested = self.publish_requested.lock(); *requested = (*requested).max(publish); } + if self.checkout.is_building() { + return; + } if self.refreshing.swap(true, Ordering::AcqRel) { return; } @@ -527,6 +533,9 @@ impl Work { let mut flag = Some(flag); loop { let result = work.checkout.refresh_callgraph(); + if work.checkout.is_building() { + return; + } if !matches!(result, Ok(None) | Err(DataError::Cancelled)) { work.notify_status_change(); } From 1bfaa045cd5bebaa7b2b7d66110098b8c6987064 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:27:20 +0100 Subject: [PATCH 005/163] mason: clarify refresh budget and build ownership comments Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit ad8f7b617fbf3511e3ac139d7f5b6bf50aff911d) (cherry picked from commit 6f10ca84de8e88384cd2c7702e0605c6cac27115) --- crates/aft/src/data_layer/checkout.rs | 6 ++++-- crates/aft/src/data_layer/refresh_budget.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/aft/src/data_layer/checkout.rs b/crates/aft/src/data_layer/checkout.rs index 7e661d8cac..155cfbdc89 100644 --- a/crates/aft/src/data_layer/checkout.rs +++ b/crates/aft/src/data_layer/checkout.rs @@ -80,7 +80,8 @@ pub(crate) struct SourceWork { /// Passes over the files: a scan reads every file whose stat does not /// prove its recorded content, and a build's scan parses what it reads. pub(crate) scans: usize, - /// Whole call-graph cold builds, excluding text-only scans and sibling admission. + /// Full graph rebuilds, not text-only scans or incremental builds from + /// another checkout's published graph. pub(crate) cold_builds: usize, /// Call-graph resolutions: a build's, or a refresh's update. pub(crate) resolutions: usize, @@ -2651,7 +2652,8 @@ impl Checkout { if let Some(reservation) = reservation { self.build_reserved(false, reservation)?; } else { - // Sibling admission already owns the build reservation. + // The outer build is reusing another checkout's publication + // and already holds the reservation. self.build_inner(false)?; } return Ok(Some(UpdateStats { diff --git a/crates/aft/src/data_layer/refresh_budget.rs b/crates/aft/src/data_layer/refresh_budget.rs index cd2d8cef13..29c7cd63d2 100644 --- a/crates/aft/src/data_layer/refresh_budget.rs +++ b/crates/aft/src/data_layer/refresh_budget.rs @@ -1,4 +1,4 @@ -//! Admission by the files an incremental engine has invalidated since publication. +//! Limits an incremental update by the files invalidated since publication. use std::collections::BTreeSet; From 04b48317cc49de0173cafd5614fd6baccf7acae1 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:31:28 +0100 Subject: [PATCH 006/163] mason: admit declaration readers before resolving bystanders Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 3ac93a8b47cd1af7fd42db9ac4242344c076bc6f) (cherry picked from commit 80f2b4090677ee26702deb384b51045429cbde79) --- crates/aft/src/data_layer/callgraph.rs | 23 +++++++++++++++++++++++ crates/aft/src/data_layer/incremental.rs | 8 ++++++++ 2 files changed, 31 insertions(+) diff --git a/crates/aft/src/data_layer/callgraph.rs b/crates/aft/src/data_layer/callgraph.rs index 8dff1ae854..bd47f24b1c 100644 --- a/crates/aft/src/data_layer/callgraph.rs +++ b/crates/aft/src/data_layer/callgraph.rs @@ -1746,6 +1746,29 @@ pub(crate) struct MethodCallInputs { const SMALL_SITE_RUN: usize = 64; impl MethodCallInputs { + /// Declaration changes already identify method readers before resolving + /// any importer. Compare raw facts on both sides: import/type targets are + /// filled by resolution and are admitted separately when they are known. + pub(crate) fn changed_parse_method_keys( + old: &CheckoutInputs, + next: &CheckoutInputs, + changed: &BTreeSet, + ) -> BTreeSet { + let mut keys = BTreeSet::new(); + for path in changed { + let before = old + .parses + .read(path) + .map(|(_, parse)| FileFacts::new(&parse)); + let after = next + .parses + .read(path) + .map(|(_, parse)| FileFacts::new(&parse)); + FileFacts::changed_keys_by_method(path, before.as_ref(), after.as_ref(), &mut keys); + } + keys + } + /// The files whose inputs the resolver computed itself since the last /// call. #[cfg(test)] diff --git a/crates/aft/src/data_layer/incremental.rs b/crates/aft/src/data_layer/incremental.rs index 1cc47e1bcd..4d0088a24f 100644 --- a/crates/aft/src/data_layer/incremental.rs +++ b/crates/aft/src/data_layer/incremental.rs @@ -808,6 +808,14 @@ impl LiveCheckout { }) .cloned() .collect::>(); + if budget.is_some() { + // A module move can invalidate thousands of method readers even + // when few paths changed. Admit those declaration readers before + // binding or resolving bystanders, not only at the sites pass. + let keys = MethodCallInputs::changed_parse_method_keys(&old, inputs, &contents); + let readers = self.reverse.sites_of(&keys); + admit(&mut budget, &readers)?; + } // Configuration files added or removed. A content change of an // existing one reaches its readers through their recorded field // answers, which are asked again below. Adding or removing one From 567d225188d15289ed68cf97a649aea71332d891 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:42:49 +0100 Subject: [PATCH 007/163] mason: hand extracted change records to the cold fallback Reuse initialized encoded parse records by their content/extractor key without forcing untouched changes to extract. The cumulative-switch test now asserts every target record is reused, including the two unpublished extracts. Add allocation-counter-free stage timings and a test-hooks control for attributing the old repeated work. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 33c693f7ffcb0ee35a092c9d2257c5743d93c72f) (cherry picked from commit 32c2c1ad45f27fd5d5bc3c6cca30092450a8501e) --- crates/aft/examples/alloc_trace_probe.rs | 18 ++++++---- crates/aft/src/data_layer/checkout.rs | 42 +++++++++++++++++++++--- crates/aft/src/data_layer/incremental.rs | 6 +++- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/crates/aft/examples/alloc_trace_probe.rs b/crates/aft/examples/alloc_trace_probe.rs index 99c8d9a9a7..4d011aeacb 100644 --- a/crates/aft/examples/alloc_trace_probe.rs +++ b/crates/aft/examples/alloc_trace_probe.rs @@ -76,6 +76,7 @@ struct Switch; static TRACE: AtomicBool = AtomicBool::new(false); static COUNTING: AtomicBool = AtomicBool::new(false); +static STAGE_TIMING: AtomicBool = AtomicBool::new(false); static LIVE: AtomicIsize = AtomicIsize::new(0); static PEAK: AtomicIsize = AtomicIsize::new(0); static STAGE_RSS: AtomicU64 = AtomicU64::new(0); @@ -429,7 +430,7 @@ static STAGE: Mutex> = Mutex::new(None); /// previous stage. Counting is opt-in because atomics on every allocation /// perturb CPU timings. C-library memory is visible only in the RSS samples. fn stage(name: &'static str) { - if !COUNTING.load(Ordering::Relaxed) { + if !STAGE_TIMING.load(Ordering::Relaxed) { return; } let mut current = STAGE.lock().unwrap(); @@ -445,9 +446,9 @@ fn stage(name: &'static str) { "refresh_stage": previous.name, "wall_s": previous.started.elapsed().as_secs_f64(), "cpu_s": cpu.0 + cpu.1 - previous.cpu.0 - previous.cpu.1, - "live_bytes_at_start": previous.live, - "peak_live_bytes": peak, - "live_bytes_at_end": live, + "live_bytes_at_start": COUNTING.load(Ordering::Relaxed).then_some(previous.live), + "peak_live_bytes": COUNTING.load(Ordering::Relaxed).then_some(peak), + "live_bytes_at_end": COUNTING.load(Ordering::Relaxed).then_some(live), "resident_peak_bytes": rss_peak, "malloc_at_end": malloc_figures(), }) @@ -846,13 +847,18 @@ fn transition(mut args: impl Iterator) { let strategy = args.next().unwrap_or_else(|| "sibling".to_string()); assert!(["sibling", "refresh", "cold"].contains(&strategy.as_str())); let profiling = std::env::var_os("PROBE_STAGES").is_some(); + let stage_timing = profiling || std::env::var_os("PROBE_STAGE_TIMING").is_some(); COUNTING.store(profiling, Ordering::Relaxed); + STAGE_TIMING.store(stage_timing, Ordering::Relaxed); #[cfg(feature = "test-hooks")] - if profiling { + if stage_timing { aft::data_layer::refresh_probe::set_observer(stage); } #[cfg(not(feature = "test-hooks"))] - assert!(!profiling, "PROBE_STAGES requires --features test-hooks"); + assert!( + !stage_timing, + "stage measurement requires --features test-hooks" + ); start_tracing(); let scratch = tempfile::tempdir().unwrap(); let store = scratch.path().join("store"); diff --git a/crates/aft/src/data_layer/checkout.rs b/crates/aft/src/data_layer/checkout.rs index 155cfbdc89..24f2e0df05 100644 --- a/crates/aft/src/data_layer/checkout.rs +++ b/crates/aft/src/data_layer/checkout.rs @@ -1974,9 +1974,30 @@ impl Checkout { Ok(StoredRecords { _snapshot: Some(snapshot), segments, + pending: self.ready_change_records(), }) } + /// A cold scan can reuse content already extracted by an aborted update. + /// Looking only at initialized values preserves the pre-extraction cutoff; + /// the scan still checks the on-disk content key before using a record. + fn ready_change_records(&self) -> HashMap> { + #[cfg(feature = "test-hooks")] + if std::env::var_os("PROBE_SKIP_HANDOFF").is_some() { + return HashMap::new(); + } + self.live + .read() + .expect("no panics while holding it") + .changes + .values() + .filter_map(|change| match change.content.peek() { + Some(Extracted::Parse(key, _, record)) => Some((key.clone(), Arc::clone(record))), + _ => None, + }) + .collect() + } + /// Waits up to [`WRITER_WAIT`] for the store's writer, and refuses it /// below the free-disk floor. The store itself refuses it, before /// anything is written, when a newer build wrote a file the newest @@ -2649,15 +2670,16 @@ impl Checkout { limit.parsed ); let started = Instant::now(); - if let Some(reservation) = reservation { - self.build_reserved(false, reservation)?; + let rebuilt = if let Some(reservation) = reservation { + self.build_reserved(false, reservation)? } else { // The outer build is reusing another checkout's publication // and already holds the reservation. - self.build_inner(false)?; - } + self.build_inner(false)? + }; return Ok(Some(UpdateStats { cold_build: true, + cold_reused_records: rebuilt.stored_records, total_ms: elapsed_ms(started), ..UpdateStats::default() })); @@ -4733,6 +4755,9 @@ struct StoredRecords { _snapshot: Option, /// Newest last. segments: Vec>, + /// Encoded parses computed by pending changes, keyed exactly as a stored + /// content record. They need not be published just to avoid parsing twice. + pending: HashMap>, } impl StoredRecords { @@ -4740,6 +4765,11 @@ impl StoredRecords { /// segment that fails to read counts as not holding it, so the file is /// parsed instead. fn record(&self, key: RecordKey) -> Option { + if let Some(record) = self.pending.get(&key.digest().to_string()) { + if let Ok(record) = record.decode() { + return Some(record); + } + } self.segments.iter().rev().find_map(|segment| { let content = ContentSegment::open(segment).ok().flatten()?; let ordinal = content.find(key).ok().flatten()?; @@ -7777,6 +7807,10 @@ pub fn other() { callee(); } "two disjoint sub-threshold passes exceed the union limit" ); assert_eq!(source_work_for_test(root).cold_builds, before + 1); + assert_eq!( + second.cold_reused_records, 1102, + "the cold scan reuses both unpublished extracts" + ); assert_eq!(checkout.status().callgraph, PlaneStatus::Ready); assert!(checkout.refresh_callgraph().unwrap().is_none()); assert_eq!(source_work_for_test(root).cold_builds, before + 1); diff --git a/crates/aft/src/data_layer/incremental.rs b/crates/aft/src/data_layer/incremental.rs index 4d0088a24f..cc28d8e306 100644 --- a/crates/aft/src/data_layer/incremental.rs +++ b/crates/aft/src/data_layer/incremental.rs @@ -56,6 +56,9 @@ use super::subjects::{affected_by, subjects, Subject}; pub struct UpdateStats { /// A broad incremental update was replaced by a stored-record cold build. pub cold_build: bool, + /// Content records reused by the cold build, including extracted changes + /// handed over by the abandoned incremental update. + pub cold_reused_records: usize, pub changed_paths: usize, /// Recorded checkout reads asked again, and how many got a different /// answer. @@ -811,7 +814,8 @@ impl LiveCheckout { if budget.is_some() { // A module move can invalidate thousands of method readers even // when few paths changed. Admit those declaration readers before - // binding or resolving bystanders, not only at the sites pass. + // binding or resolving other files, rather than waiting until + // method-call sites are processed. let keys = MethodCallInputs::changed_parse_method_keys(&old, inputs, &contents); let readers = self.reverse.sites_of(&keys); admit(&mut budget, &readers)?; From d5fa960c508995c28181c27adc38ea1d7b3c495e Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:26:46 +0100 Subject: [PATCH 008/163] mason: preserve cold-build lifecycle when refreshing Reserved builds wait out and release any in-flight incremental engine before scanning. Cold fallbacks also schedule meaning fill after publication, as the existing full-build path does. Clarify probe timing controls and initialized-record reuse. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 8a1071417a3860edaf19cac00b6fa6524363491a) (cherry picked from commit 334e16b009e1648a5f07b308b4ae3c568bf655db) --- crates/aft/examples/alloc_trace_probe.rs | 6 ++++++ crates/aft/src/data_layer/checkout.rs | 13 +++++++++++-- crates/aft/src/data_layer/runtime.rs | 8 ++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/aft/examples/alloc_trace_probe.rs b/crates/aft/examples/alloc_trace_probe.rs index 4d011aeacb..32612e97c6 100644 --- a/crates/aft/examples/alloc_trace_probe.rs +++ b/crates/aft/examples/alloc_trace_probe.rs @@ -55,6 +55,12 @@ //! refreshes print the block reads they made through the layer's cache. //! With `PROBE_TRIM=1`, `refresh` hands the allocator's free memory back //! before the restart's refresh. +//! +//! `transition` compares two exported source trees. `PROBE_STAGES=1` counts +//! the live Rust heap at each stage; `PROBE_STAGE_TIMING=1` reports stage +//! wall/CPU times without those counting atomics. Both require `test-hooks`. +//! `PROBE_SKIP_HANDOFF=1` disables pending-record reuse for a timing control; +//! `PROBE_COMPARE=1` compares every resulting engine record with a cold build. #![allow(clippy::unwrap_used, clippy::print_stdout, clippy::print_stderr)] diff --git a/crates/aft/src/data_layer/checkout.rs b/crates/aft/src/data_layer/checkout.rs index 24f2e0df05..c066ee2ac8 100644 --- a/crates/aft/src/data_layer/checkout.rs +++ b/crates/aft/src/data_layer/checkout.rs @@ -1510,6 +1510,14 @@ impl Checkout { measure: bool, reservation: BuildReservation, ) -> Result { + // A refresh already inside the engine finishes first. The reservation + // blocks new refreshes, so no decoded engine overlaps the cold scan. + let mut engine = self + .engine + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + *engine = None; + drop(engine); let _alloc_label = crate::alloc_trace::scope(crate::alloc_trace::Label::INDEX_BUILD, Some(&self.root)); let result = self.build_inner(measure); @@ -1979,8 +1987,9 @@ impl Checkout { } /// A cold scan can reuse content already extracted by an aborted update. - /// Looking only at initialized values preserves the pre-extraction cutoff; - /// the scan still checks the on-disk content key before using a record. + /// Reading this cache must not start parsing: a batch rejected before + /// extraction should reach the cold scan unparsed. The scan looks up each + /// record by its on-disk content and extractor key. fn ready_change_records(&self) -> HashMap> { #[cfg(feature = "test-hooks")] if std::env::var_os("PROBE_SKIP_HANDOFF").is_some() { diff --git a/crates/aft/src/data_layer/runtime.rs b/crates/aft/src/data_layer/runtime.rs index 371aaa78f9..0ea0918536 100644 --- a/crates/aft/src/data_layer/runtime.rs +++ b/crates/aft/src/data_layer/runtime.rs @@ -540,6 +540,14 @@ impl Work { work.notify_status_change(); } if matches!(result, Ok(Some(_))) { + if result + .as_ref() + .is_ok_and(|stats| stats.as_ref().is_some_and(|stats| stats.cold_build)) + { + // A cold fallback already published its content, just + // as start_build does, so meaning search can fill it. + work.start_fill(); + } continue; } match &result { From 74157da1cc2e12f3298429a4d2814d389a724e47 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:04:11 +0100 Subject: [PATCH 009/163] mason: clarify semantic fill after a cold publication Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 0e24df2bee14b21b027dd4f0268422b233530dcf) (cherry picked from commit 9855b96be8a754973cf70ee827f2247cb8b4278d) --- crates/aft/src/data_layer/runtime.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/aft/src/data_layer/runtime.rs b/crates/aft/src/data_layer/runtime.rs index 0ea0918536..a8b14d8e33 100644 --- a/crates/aft/src/data_layer/runtime.rs +++ b/crates/aft/src/data_layer/runtime.rs @@ -545,7 +545,8 @@ impl Work { .is_ok_and(|stats| stats.as_ref().is_some_and(|stats| stats.cold_build)) { // A cold fallback already published its content, just - // as start_build does, so meaning search can fill it. + // as start_build does, so semantic search can embed and + // index the newly published content. work.start_fill(); } continue; From 06e82102bbfef52a3a5278afa3ca13425c4e7290 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:03:29 +0100 Subject: [PATCH 010/163] mason: resolve exact computed Rust include inputs (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit ac29aaa726c41cf7fc9ef0cd9c40a302ff3dcaa1) (cherry picked from commit 0f0f5c74beba96968c504b5f87ea3f5010b69188) --- .alfonso/release-notes/next/issue-1425.md | 1 + crates/aft/src/lsp/completed_rust_check.rs | 494 +++++++++++++--- .../src/lsp/completed_rust_check/inputs.rs | 559 ++++++++++++++++++ 3 files changed, 960 insertions(+), 94 deletions(-) create mode 100644 .alfonso/release-notes/next/issue-1425.md create mode 100644 crates/aft/src/lsp/completed_rust_check/inputs.rs diff --git a/.alfonso/release-notes/next/issue-1425.md b/.alfonso/release-notes/next/issue-1425.md new file mode 100644 index 0000000000..f1805cf9de --- /dev/null +++ b/.alfonso/release-notes/next/issue-1425.md @@ -0,0 +1 @@ +Rust diagnostics now handle literal `concat!` include paths and local fixture macros with literal arguments. Unresolved includes name their source file. Thanks @elmagow. (#1425) diff --git a/crates/aft/src/lsp/completed_rust_check.rs b/crates/aft/src/lsp/completed_rust_check.rs index a028299077..dad0c4d293 100644 --- a/crates/aft/src/lsp/completed_rust_check.rs +++ b/crates/aft/src/lsp/completed_rust_check.rs @@ -13,10 +13,12 @@ use serde::{Deserialize, Serialize}; use super::diagnostics::{DiagnosticSeverity, StoredDiagnostic}; +mod inputs; +use inputs::{SourceInputs, UnresolvedInput}; + pub(crate) const BUDGET: Duration = Duration::from_secs(2); -// The first completed check has no trusted hash ledger yet. Parsing literal -// include dependencies is a one-time cost; later requests reuse the ledger and -// retain the much smaller serving budget. +// The first completed check has no trusted hash ledger yet. Later requests +// reuse content hashes, but still resolve macros against current source files. const INITIAL_CAPTURE_BUDGET: Duration = Duration::from_secs(15); // Filesystem timestamps can round a write down to a tick before the wall-clock // instant at which it actually happened. Two seconds covers coarse timestamp @@ -248,84 +250,7 @@ fn workspace_packages(root: &Path, deadline: Instant) -> Option> { Some(packages) } -fn source_inputs( - path: &Path, - bytes: &[u8], - deadline: Instant, -) -> Option<(Vec, Vec)> { - let text = std::str::from_utf8(bytes).ok()?; - if !text.contains("include") && !text.contains("env") { - return Some((Vec::new(), Vec::new())); - } - let mut parser = tree_sitter::Parser::new(); - parser - .set_language(&tree_sitter_rust::LANGUAGE.into()) - .ok()?; - let mut progress = |_: &tree_sitter::ParseState| { - if Instant::now() >= deadline { - std::ops::ControlFlow::Break(()) - } else { - std::ops::ControlFlow::Continue(()) - } - }; - let tree = parser.parse_with_options( - &mut |offset, _| bytes.get(offset..).unwrap_or_default(), - None, - Some(tree_sitter::ParseOptions::new().progress_callback(&mut progress)), - )?; - expired(deadline)?; - let mut nodes = vec![tree.root_node()]; - let mut included = Vec::new(); - let mut env_keys = Vec::new(); - while let Some(node) = nodes.pop() { - expired(deadline)?; - for index in 0..u32::try_from(node.child_count()).ok()? { - let child = node.child(index)?; - nodes.push(child); - if !matches!(child.kind(), "identifier" | "scoped_identifier") { - continue; - } - let name = child.utf8_text(bytes).ok()?.rsplit("::").next()?; - if !matches!( - name, - "include" | "include_str" | "include_bytes" | "env" | "option_env" - ) { - continue; - } - if node.child(index + 1).is_none_or(|bang| bang.kind() != "!") { - continue; - } - let tokens = node.child(index + 2)?; - let mut cursor = tokens.walk(); - let values = tokens - .named_children(&mut cursor) - .filter(|n| !matches!(n.kind(), "line_comment" | "block_comment")) - .collect::>(); - // A generated/dynamic include path is uncertain. Do not guess what - // concat!, env!, or a procedural macro will read from disk. - if values.is_empty() || (values.len() != 1 && !matches!(name, "env" | "option_env")) { - return None; - } - let literal = values[0].utf8_text(bytes).ok()?; - let relative = match values[0].kind() { - "string_literal" => serde_json::from_str::(literal).ok()?, - "raw_string_literal" => { - let start = literal.find('"')?; - let end = literal.rfind('"')?; - literal.get(start + 1..end)?.to_string() - } - _ => return None, - }; - if matches!(name, "env" | "option_env") { - env_keys.push(relative); - } else { - included.push(canonical(&path.parent()?.join(relative))?); - } - } - } - Some((included, env_keys)) -} - +#[cfg(test)] pub(crate) fn fingerprint( root: &Path, runtime: String, @@ -334,6 +259,17 @@ pub(crate) fn fingerprint( // When capturing a check's inputs after reading its begin event, no file // or directory may have changed since the reader observed that event. began: Option, +) -> Option { + fingerprint_with_reason(root, runtime, previous, deadline, began, &mut None) +} + +fn fingerprint_with_reason( + root: &Path, + runtime: String, + previous: Option<&Fingerprint>, + deadline: Instant, + began: Option, + unresolved: &mut Option, ) -> Option { let latest_safe_time = match began { Some(begin) => Some(begin.checked_sub(CHECK_BEGIN_TIMESTAMP_MARGIN)?), @@ -348,6 +284,7 @@ pub(crate) fn fingerprint( started: Instant::now(), }; let mut hashed = 0; + let mut source_inputs = SourceInputs::default(); let mut pending = workspace_packages(root, deadline)?; let mut inputs = vec![root.join("Cargo.toml")]; let extra_inputs = previous.map_or_else(Vec::new, |p| p.extra_inputs.clone()); @@ -435,10 +372,21 @@ pub(crate) fn fingerprint( } } } - while let Some(path) = inputs.pop() { + loop { + if inputs.is_empty() { + for path in source_inputs.resolve(root, deadline, unresolved)? { + expired(deadline)?; + if !files.contains_key(&path) || source_inputs.needs_parse(&path) { + inputs.push(path); + } + } + } + let Some(path) = inputs.pop() else { + break; + }; report.last = path.clone(); expired(deadline)?; - if files.contains_key(&path) { + if files.contains_key(&path) && !source_inputs.needs_parse(&path) { continue; } let metadata = fs::symlink_metadata(&path).ok()?; @@ -479,6 +427,7 @@ pub(crate) fn fingerprint( } } let old = previous.and_then(|p| p.files.get(&path)); + let mut source_bytes = None; // Manifests are always read, because their path dependencies also need // validating. Index hashes with no inode-change proof cannot be reused. let manifest = path.file_name().is_some_and(|name| name == "Cargo.toml"); @@ -499,11 +448,24 @@ pub(crate) fn fingerprint( return None; } current.hash = blake3::hash(&bytes).to_hex().to_string(); - if path.extension().is_some_and(|extension| extension == "rs") { - (current.included, current.env_keys) = source_inputs(&path, &bytes, deadline)?; - } + source_bytes = Some(bytes); hashed += 1; } + if source_inputs.needs_parse(&path) { + // Macro invocations in another source file can change the paths a + // definition reads, even when the definition's own bytes did not. + let bytes = source_bytes.or_else(|| read_bounded(&path, deadline))?; + let observed = stamp(&fs::metadata(&path).ok()?)?; + if (observed.size, observed.modified, observed.changed) + != (current.size, current.modified, current.changed) + { + return None; + } + let (included, env_keys) = + source_inputs.add(root, &path, &bytes, deadline, unresolved)?; + current.included = included; + current.env_keys = env_keys; + } inputs.extend(current.included.clone()); files.insert(path, current); } @@ -704,6 +666,17 @@ impl Runtime { previous: Option<&Fingerprint>, deadline: Instant, began: Option, + ) -> Option { + self.capture_with_reason(root, previous, deadline, began, &mut None) + } + + fn capture_with_reason( + &self, + root: &Path, + previous: Option<&Fingerprint>, + deadline: Instant, + began: Option, + unresolved: &mut Option, ) -> Option { let runtime = self.hash(root, deadline)?; let mut seed = previous.cloned().unwrap_or(Fingerprint { @@ -738,7 +711,8 @@ impl Runtime { } seed.extra_inputs.sort(); seed.extra_inputs.dedup(); - let captured = fingerprint(root, runtime, Some(&seed), deadline, began)?; + let captured = + fingerprint_with_reason(root, runtime, Some(&seed), deadline, began, unresolved)?; for key in captured .files .values() @@ -1224,11 +1198,13 @@ impl CompletedRustCheck { // Capture before spawning, not after receiving a progress event. This // also works on platforms without inode change timestamps. self.reports.clear(); - let before = self.runtime.capture( + let mut unresolved = None; + let before = self.runtime.capture_with_reason( &self.root, self.saved.as_ref().map(|saved| &saved.fingerprint), deadline, None, + &mut unresolved, ); let mut args = vec![ "check".to_string(), @@ -1265,14 +1241,19 @@ impl CompletedRustCheck { self.reports = reports; let Some(before) = before else { self.abort(); + if let Some(unresolved) = unresolved { + return Err(format!("aft_inspect: {}", unresolved.note(&self.root))); + } return Err("aft_inspect: cargo check completed, but Rust inputs cannot be fingerprinted; diagnostics are provisional".into()); }; - let after = self - .runtime - .capture(&self.root, Some(&before), deadline, None) - .ok_or_else(|| { - "aft_inspect: cannot validate Rust inputs after cargo check; retry".to_string() - })?; + let after = + self.runtime + .capture_with_reason(&self.root, Some(&before), deadline, None, &mut unresolved) + .ok_or_else(|| match unresolved { + Some(unresolved) => format!("aft_inspect: {}", unresolved.note(&self.root)), + None => "aft_inspect: cannot validate Rust inputs after cargo check; retry" + .to_string(), + })?; if cancelled() || before.files != after.files || before.digest != after.digest { self.abort(); return Err("aft_inspect: Rust inputs changed during cargo check; retry".into()); @@ -2438,6 +2419,331 @@ mod tests { assert!(valid(&cache).is_none()); } + fn capture_inputs(cache: &CompletedRustCheck) -> Option { + fingerprint( + &cache.root, + "v1".into(), + None, + Instant::now() + BUDGET, + None, + ) + } + + fn unresolved_input(cache: &CompletedRustCheck) -> String { + let mut unresolved = None; + assert!(fingerprint_with_reason( + &cache.root, + "v1".into(), + None, + Instant::now() + BUDGET, + None, + &mut unresolved, + ) + .is_none()); + unresolved + .expect("the source causing uncertainty must be named") + .note(&cache.root) + } + + #[test] + fn literal_concat_includes_track_exact_files() { + for name in ["include", "include_str", "include_bytes"] { + let (_temp, mut cache) = fixture(); + write(&cache.root, "member/src/data/a.txt", "before"); + write(&cache.root, "member/src/data/unused.txt", "unrelated"); + write( + &cache.root, + "member/src/lib.rs", + &format!("const DATA: &str = {name}!(concat!(r#\"data/\"#, \"a\", \".txt\",));\n"), + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write( + &cache.root, + "member/src/data/unused.txt", + "changed unrelated", + ); + assert!( + valid(&cache).is_some(), + "only the exact concat path is an input" + ); + write(&cache.root, "member/src/data/a.txt", "after!"); + assert!( + valid(&cache).is_none(), + "{name} input must invalidate the check" + ); + } + } + + #[test] + fn concat_manifest_dir_uses_nearest_member_manifest() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "member/data/a.txt", "before"); + write( + &cache.root, + "member/src/nested/inputs.rs", + r#" + const DATA: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/", "a.txt")); + "#, + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(cache + .saved + .as_ref() + .unwrap() + .fingerprint + .files + .contains_key(&cache.root.join("member/data/a.txt"))); + assert!(valid(&cache).is_some()); + write(&cache.root, "member/data/a.txt", "after!"); + assert!(valid(&cache).is_none()); + } + + #[test] + fn reporter_fixture_macro_tracks_all_literal_invocations_across_files() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "tests/fixtures/a.txt", "one"); + write(&cache.root, "tests/fixtures/b.txt", "two"); + write(&cache.root, "tests/fixtures/unread.txt", "not an input"); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { + ($fixture:literal) => { include_str!(concat!("../../tests/fixtures/", $fixture)) }; + } + const A: &str = fixture!("a.txt"); + "#, + ); + write( + &cache.root, + "member/src/calls.rs", + "const B: &str = fixture!(\"b.txt\");\n", + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write( + &cache.root, + "tests/fixtures/unread.txt", + "still not an input", + ); + assert!(valid(&cache).is_some()); + for (file, original) in [("a.txt", "one"), ("b.txt", "two")] { + write(&cache.root, &format!("tests/fixtures/{file}"), "changed"); + assert!( + valid(&cache).is_none(), + "every resolved invocation is an input" + ); + write(&cache.root, &format!("tests/fixtures/{file}"), original); + } + write(&cache.root, "tests/fixtures/new.txt", "new input"); + write( + &cache.root, + "member/src/calls.rs", + "const B: &str = fixture!(\"new.txt\");\n", + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write(&cache.root, "tests/fixtures/new.txt", "changed new input"); + assert!( + valid(&cache).is_none(), + "unchanged definitions must use the current calls" + ); + } + + #[test] + fn macro_paths_and_manifest_dir_are_relative_to_the_invocation() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "member/tests/data/a.txt", "before"); + write(&cache.root, "member/data/b.txt", "before"); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! relative { ($name:literal) => { include_bytes!(concat!("data/", $name)) }; } + macro_rules! absolute { ($name:literal) => { include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/", $name)) }; } + "#, + ); + write( + &cache.root, + "member/tests/check.rs", + r#" + const A: &[u8] = relative!("a.txt"); + const B: &str = absolute!("b.txt"); + "#, + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write(&cache.root, "member/tests/data/a.txt", "after!"); + assert!(valid(&cache).is_none()); + write(&cache.root, "member/tests/data/a.txt", "before"); + write(&cache.root, "member/data/b.txt", "after!"); + assert!(valid(&cache).is_none()); + } + + #[test] + fn unused_fixture_macro_contributes_no_include_input() { + let (_temp, cache) = fixture(); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { ($fixture:literal) => { include_str!(concat!("missing/", $fixture)) }; } + "#, + ); + assert!(capture_inputs(&cache).is_some()); + } + + #[test] + fn unresolved_include_names_its_source_with_approved_wording() { + let (_temp, cache) = fixture(); + for source in [ + "const DATA: &str = include_str!(some_macro!());", + "const DATA: &str = include_str!(concat!(\"data/\", some_macro!()));", + "const DATA: &str = include_str!(concat!(\"data/\", \"missing.txt\"));", + ] { + write(&cache.root, "member/src/lib.rs", source); + assert_eq!(unresolved_input(&cache), "cargo check completed, but member/src/lib.rs includes a file whose path can't be resolved; diagnostics are provisional"); + } + } + + #[test] + fn non_literal_environment_keys_name_their_source_with_approved_wording() { + let (_temp, cache) = fixture(); + for source in [ + "const DATA: &str = env!(key!());", + "const DATA: Option<&str> = option_env!(concat!(\"A\", \"B\"));", + "const DATA: &str = include_str!(concat!(env!(key!()), \"/data.txt\"));", + ] { + write(&cache.root, "member/src/lib.rs", source); + assert_eq!(unresolved_input(&cache), "cargo check completed, but member/src/lib.rs reads an environment variable whose name can't be resolved; diagnostics are provisional"); + } + } + + #[test] + fn unresolved_fixture_invocations_stay_provisional() { + for invocation in [ + "fixture!(name)", + "fixture!(concat!(\"a\", \".txt\"))", + "forward!(fixture!(\"a.txt\"))", + "macro_rules! forward { () => { fixture!(\"a.txt\") }; }", + "forward!(fixture)", + "use fixture as alias; alias!(\"a.txt\")", + ] { + let (_temp, cache) = fixture(); + write(&cache.root, "member/src/data/a.txt", "input"); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { ($fixture:literal) => { include_str!(concat!("data/", $fixture)) }; } + "#, + ); + write(&cache.root, "member/src/calls.rs", invocation); + assert_eq!(unresolved_input(&cache), "cargo check completed, but member/src/lib.rs includes a file whose path can't be resolved; diagnostics are provisional", "{invocation}"); + } + } + + #[test] + fn literal_macro_traversal_is_resolved_exactly_or_refused_outside_workspace() { + let (temp, mut cache) = fixture(); + write(&cache.root, "member/src/data/placeholder.txt", "not read"); + write(&cache.root, "member/src/outside.txt", "before"); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { ($fixture:literal) => { include_str!(concat!("data/", $fixture)) }; } + const DATA: &str = fixture!("../outside.txt"); + "#, + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write(&cache.root, "member/src/outside.txt", "after!"); + assert!(valid(&cache).is_none()); + fs::write(temp.path().join("outside.txt"), "external").unwrap(); + write( + &cache.root, + "member/src/calls.rs", + "const DATA: &str = fixture!(\"../../../../outside.txt\");\n", + ); + assert_eq!(unresolved_input(&cache), "cargo check completed, but member/src/lib.rs includes a file whose path can't be resolved; diagnostics are provisional"); + } + + #[cfg(unix)] + #[test] + fn computed_include_refuses_symlink_components() { + let (temp, cache) = fixture(); + let input = temp.path().join("external.txt"); + fs::write(&input, "external").unwrap(); + fs::create_dir_all(cache.root.join("target")).unwrap(); + std::os::unix::fs::symlink(input, cache.root.join("target/link.txt")).unwrap(); + write( + &cache.root, + "member/src/lib.rs", + r#" + const DATA: &str = include_str!(concat!("../../target/", "link.txt")); + "#, + ); + assert_eq!(unresolved_input(&cache), "cargo check completed, but member/src/lib.rs includes a file whose path can't be resolved; diagnostics are provisional"); + } + + #[test] + fn complex_fixture_matchers_stay_provisional_when_invoked() { + for definition in [ + r#"macro_rules! fixture { ($($fixture:literal),*) => { include_str!(concat!("data/", $($fixture),*)) }; }"#, + r#"macro_rules! fixture { ($fixture:literal) => { include_str!(concat!("data/", $fixture)) }; () => { "empty" }; }"#, + ] { + let (_temp, cache) = fixture(); + write( + &cache.root, + "member/src/lib.rs", + &format!("{definition}\nconst DATA: &str = fixture!(\"a.txt\");"), + ); + assert_eq!(unresolved_input(&cache), "cargo check completed, but member/src/lib.rs includes a file whose path can't be resolved; diagnostics are provisional"); + } + } + + #[test] + fn macro_calls_in_included_rust_with_non_rust_extension_are_scanned() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "member/src/data/a.txt", "before"); + write( + &cache.root, + "member/src/calls.inc", + "const DATA: &str = fixture!(\"a.txt\");", + ); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { ($fixture:literal) => { include_str!(concat!("data/", $fixture)) }; } + include!(concat!("calls", ".inc")); + "#, + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write(&cache.root, "member/src/data/a.txt", "after!"); + assert!(valid(&cache).is_none()); + } + + #[test] + fn concat_paths_decode_rust_string_escapes() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "member/src/data/a.txt", "before"); + write( + &cache.root, + "member/src/lib.rs", + r#" + const DATA: &str = include_str!(concat!("d\x61ta/", "\u{61}.txt")); + "#, + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write(&cache.root, "member/src/data/a.txt", "after!"); + assert!(valid(&cache).is_none()); + } + #[test] fn declared_build_input_directory_addition_invalidates_completed_check() { let (_temp, mut cache) = fixture(); diff --git a/crates/aft/src/lsp/completed_rust_check/inputs.rs b/crates/aft/src/lsp/completed_rust_check/inputs.rs new file mode 100644 index 0000000000..7ae6804e49 --- /dev/null +++ b/crates/aft/src/lsp/completed_rust_check/inputs.rs @@ -0,0 +1,559 @@ +//! Resolve compiler inputs only when the source proves every concrete path. +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use super::{canonical, expired}; + +const INCLUDE_PROVISIONAL: &str = "cargo check completed, but {file} includes a file whose path can't be resolved; diagnostics are provisional"; +const ENV_PROVISIONAL: &str = "cargo check completed, but {file} reads an environment variable whose name can't be resolved; diagnostics are provisional"; + +pub(super) struct UnresolvedInput { + path: PathBuf, + environment: bool, +} + +impl UnresolvedInput { + pub(super) fn note(&self, root: &Path) -> String { + let file = self.path.strip_prefix(root).unwrap_or(&self.path); + let template = if self.environment { + ENV_PROVISIONAL + } else { + INCLUDE_PROVISIONAL + }; + template.replace("{file}", &file.to_string_lossy().replace('\\', "/")) + } +} + +#[derive(Clone, Debug)] +enum Token { + Literal(String), + Text(String), + Group(Vec), +} + +impl Token { + fn text(&self, expected: &str) -> bool { + matches!(self, Self::Text(text) if text == expected) + } +} + +fn tokens(node: tree_sitter::Node<'_>, bytes: &[u8], deadline: Instant) -> Option> { + let mut result = Vec::new(); + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + expired(deadline)?; + let text = child.utf8_text(bytes).ok()?; + match child.kind() { + "line_comment" | "block_comment" | "(" | ")" | "[" | "]" | "{" | "}" => {} + "string_literal" => result.push(Token::Literal(string_literal(text, deadline)?)), + "raw_string_literal" => result.push(Token::Literal( + text.get(text.find('"')? + 1..text.rfind('"')?)? + .replace("\r\n", "\n"), + )), + "token_tree" | "token_tree_pattern" => { + result.push(Token::Group(tokens(child, bytes, deadline)?)) + } + "token_binding" => result.extend(tokens(child, bytes, deadline)?), + _ => result.push(Token::Text(text.into())), + } + } + Some(result) +} + +fn arguments(tokens: &[Token], deadline: Instant) -> Option> { + let mut args = Vec::new(); + let mut start = 0; + for (index, token) in tokens.iter().enumerate() { + expired(deadline)?; + if token.text(",") { + if start == index { + return None; + } + args.push(&tokens[start..index]); + start = index + 1; + } + } + if start < tokens.len() { + args.push(&tokens[start..]); + } + Some(args) +} + +fn manifest_dir(path: &Path, deadline: Instant) -> Option { + for parent in path.parent()?.ancestors() { + expired(deadline)?; + match fs::symlink_metadata(parent.join("Cargo.toml")) { + Ok(meta) if meta.is_file() => return Some(parent.to_str()?.into()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + _ => return None, + } + } + None +} + +fn expression( + tokens: &[Token], + path: &Path, + bindings: &BTreeMap, + deadline: Instant, +) -> Option { + expired(deadline)?; + match tokens { + [Token::Literal(value)] => Some(value.clone()), + [Token::Text(name)] => bindings.get(name).cloned(), + [name, bang, Token::Group(args)] if name.text("concat") && bang.text("!") => { + let mut value = String::new(); + for arg in arguments(args, deadline)? { + value.push_str(&expression(arg, path, bindings, deadline)?); + } + Some(value) + } + [name, bang, Token::Group(args)] if name.text("env") && bang.text("!") => { + match arguments(args, deadline)?.as_slice() { + [[Token::Literal(key)]] if key == "CARGO_MANIFEST_DIR" => { + manifest_dir(path, deadline) + } + _ => None, + } + } + _ => None, + } +} + +// Canonicalization alone would hide a symlink in an include's parent directory. +// Check every component first, including components cancelled by `..`. +fn include_path(root: &Path, source: &Path, relative: &str, deadline: Instant) -> Option { + let joined = source.parent()?.join(relative); + let mut current = PathBuf::new(); + for component in joined.components() { + expired(deadline)?; + current.push(component); + if fs::symlink_metadata(¤t).ok()?.is_symlink() { + return None; + } + } + let path = canonical(&joined)?; + (path.starts_with(root) && path.is_file()).then_some(path) +} + +struct Definition { + name: String, + path: PathBuf, + parameters: Option>, + includes: Vec<(Vec, bool)>, +} + +struct Invocation { + name: String, + path: PathBuf, + args: Vec, + nested: bool, +} + +#[derive(Default)] +pub(super) struct SourceInputs { + definitions: Vec, + invocations: Vec, + references: Vec, + rust_sources: BTreeSet, + parsed: BTreeSet, +} + +impl SourceInputs { + pub(super) fn needs_parse(&self, path: &Path) -> bool { + !self.parsed.contains(path) + && (path.extension().is_some_and(|extension| extension == "rs") + || self.rust_sources.contains(path)) + } + + fn refuse( + path: &Path, + environment: bool, + unresolved: &mut Option, + ) -> Option { + *unresolved = Some(UnresolvedInput { + path: path.into(), + environment, + }); + None + } +} + +impl SourceInputs { + pub(super) fn add( + &mut self, + root: &Path, + path: &Path, + bytes: &[u8], + deadline: Instant, + unresolved: &mut Option, + ) -> Option<(Vec, Vec)> { + std::str::from_utf8(bytes).ok()?; + self.parsed.insert(path.into()); + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&tree_sitter_rust::LANGUAGE.into()) + .ok()?; + let mut progress = |_: &tree_sitter::ParseState| { + if Instant::now() >= deadline { + std::ops::ControlFlow::Break(()) + } else { + std::ops::ControlFlow::Continue(()) + } + }; + let tree = parser.parse_with_options( + &mut |offset, _| bytes.get(offset..).unwrap_or_default(), + None, + Some(tree_sitter::ParseOptions::new().progress_callback(&mut progress)), + )?; + let mut nodes = vec![tree.root_node()]; + let mut included = Vec::new(); + let mut env_keys = Vec::new(); + while let Some(node) = nodes.pop() { + expired(deadline)?; + if node.kind() == "macro_definition" { + let name = node + .child_by_field_name("name")? + .utf8_text(bytes) + .ok()? + .into(); + let mut cursor = node.walk(); + let mut rule = None; + let mut single = true; + for child in node.named_children(&mut cursor) { + expired(deadline)?; + if child.kind() == "macro_rule" { + if rule.is_some() { + single = false; + } + rule = Some(child); + } + } + let mut definition = Definition { + name, + path: path.into(), + parameters: None, + includes: Vec::new(), + }; + if single { + if let Some(left) = rule.and_then(|rule| rule.child_by_field_name("left")) { + definition.parameters = + parameters(&tokens(left, bytes, deadline)?, deadline); + } + } + let index = self.definitions.len(); + self.definitions.push(definition); + let mut cursor = node.walk(); + for child in node.named_children(&mut cursor) { + expired(deadline)?; + if child.kind() == "macro_rule" { + let right = child.child_by_field_name("right")?; + self.scan( + root, + path, + &tokens(right, bytes, deadline)?, + Some(index), + false, + deadline, + &mut included, + &mut env_keys, + unresolved, + )?; + } + } + continue; + } + if node.kind() == "macro_invocation" { + self.scan( + root, + path, + &tokens(node, bytes, deadline)?, + None, + false, + deadline, + &mut included, + &mut env_keys, + unresolved, + )?; + continue; + } + if node.kind() == "identifier" { + self.references.push(node.utf8_text(bytes).ok()?.into()); + } + let mut cursor = node.walk(); + for child in node.named_children(&mut cursor) { + expired(deadline)?; + nodes.push(child); + } + } + Some((included, env_keys)) + } + + #[allow(clippy::too_many_arguments)] + fn scan( + &mut self, + root: &Path, + path: &Path, + tokens: &[Token], + definition: Option, + nested: bool, + deadline: Instant, + included: &mut Vec, + env_keys: &mut Vec, + unresolved: &mut Option, + ) -> Option<()> { + let mut index = 0; + while index < tokens.len() { + expired(deadline)?; + if let [Token::Text(name), bang, Token::Group(args), ..] = &tokens[index..] { + if bang.text("!") { + let name = name.rsplit("::").next()?; + match name { + "include" | "include_str" | "include_bytes" => { + if !environment_inputs(args, env_keys, deadline)? { + return Self::refuse(path, true, unresolved); + } + let arguments = arguments(args, deadline)?; + let [args] = arguments.as_slice() else { + return Self::refuse(path, false, unresolved); + }; + // A prefix is not containment: an unknown tail can + // contain `..`. Expand proven literal calls instead. + if let Some(definition) = definition { + if nested { + self.definitions[definition].parameters = None; + } + self.definitions[definition] + .includes + .push((args.to_vec(), name == "include")); + } else { + let empty = BTreeMap::new(); + let target = + expression(args, path, &empty, deadline).and_then(|relative| { + include_path(root, path, &relative, deadline) + }); + match target { + Some(target) => { + if name == "include" { + self.rust_sources.insert(target.clone()); + } + included.push(target); + } + None => return Self::refuse(path, false, unresolved), + } + } + } + "env" | "option_env" => { + let args = arguments(args, deadline)?; + match args.first().copied() { + Some([Token::Literal(key)]) => env_keys.push(key.clone()), + _ => return Self::refuse(path, true, unresolved), + } + } + _ => { + self.invocations.push(Invocation { + name: name.into(), + path: path.into(), + args: args.clone(), + nested: nested || definition.is_some(), + }); + self.scan( + root, path, args, definition, true, deadline, included, env_keys, + unresolved, + )?; + } + } + index += 3; + continue; + } + } + if let Token::Group(group) = &tokens[index] { + self.scan( + root, path, group, definition, nested, deadline, included, env_keys, unresolved, + )?; + } else if let Token::Text(name) = &tokens[index] { + self.references.push(name.clone()); + } + index += 1; + } + Some(()) + } + + pub(super) fn resolve( + &mut self, + root: &Path, + deadline: Instant, + unresolved: &mut Option, + ) -> Option> { + let mut included = Vec::new(); + for definition in &self.definitions { + expired(deadline)?; + if definition.includes.is_empty() { + continue; + } + for name in &self.references { + expired(deadline)?; + // Passing a macro by name (including a `use ... as ...` + // alias) can hide calls from this deliberately small resolver. + if name.rsplit("::").next() == Some(&definition.name) { + return Self::refuse(&definition.path, false, unresolved); + } + } + for invocation in &self.invocations { + expired(deadline)?; + if invocation.name != definition.name { + continue; + } + let Some(parameters) = definition + .parameters + .as_ref() + .filter(|_| !invocation.nested) + else { + return Self::refuse(&definition.path, false, unresolved); + }; + let Some(args) = arguments(&invocation.args, deadline) + .filter(|args| args.len() == parameters.len()) + else { + return Self::refuse(&definition.path, false, unresolved); + }; + let mut bindings = BTreeMap::new(); + for (parameter, arg) in parameters.iter().zip(args) { + expired(deadline)?; + let [Token::Literal(value)] = arg else { + return Self::refuse(&definition.path, false, unresolved); + }; + bindings.insert(parameter.clone(), value.clone()); + } + for (input, rust_source) in &definition.includes { + expired(deadline)?; + let target = expression(input, &invocation.path, &bindings, deadline).and_then( + |relative| include_path(root, &invocation.path, &relative, deadline), + ); + match target { + Some(target) => { + if *rust_source { + self.rust_sources.insert(target.clone()); + } + included.push(target); + } + None => return Self::refuse(&definition.path, false, unresolved), + } + } + } + } + Some(included) + } +} + +fn parameters(tokens: &[Token], deadline: Instant) -> Option> { + let mut names = Vec::new(); + for arg in arguments(tokens, deadline)? { + expired(deadline)?; + match arg { + [Token::Text(name), colon, kind] + if name.starts_with('$') + && colon.text(":") + && (kind.text("literal") || kind.text("expr")) => + { + if names.contains(name) { + return None; + } + names.push(name.clone()); + } + _ => return None, + } + } + Some(names) +} + +fn environment_inputs(tokens: &[Token], keys: &mut Vec, deadline: Instant) -> Option { + let mut index = 0; + while index < tokens.len() { + expired(deadline)?; + if let [name, bang, Token::Group(args), ..] = &tokens[index..] { + if (name.text("env") || name.text("option_env")) && bang.text("!") { + match arguments(args, deadline)?.first().copied() { + Some([Token::Literal(key)]) => keys.push(key.clone()), + _ => return Some(false), + } + index += 3; + continue; + } + } + if let Token::Group(args) = &tokens[index] { + if !environment_inputs(args, keys, deadline)? { + return Some(false); + } + } + index += 1; + } + Some(true) +} + +fn string_literal(text: &str, deadline: Instant) -> Option { + let normalized = text + .strip_prefix('"')? + .strip_suffix('"')? + .replace("\r\n", "\n"); + let mut chars = normalized.chars().peekable(); + let mut value = String::new(); + while let Some(ch) = chars.next() { + expired(deadline)?; + if ch != '\\' { + value.push(ch); + continue; + } + value.push(match chars.next()? { + '\\' => '\\', + '\'' => '\'', + '"' => '"', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + '0' => '\0', + 'x' => { + let code = chars.next()?.to_digit(16)? * 16 + chars.next()?.to_digit(16)?; + if code > 0x7f { + return None; + } + char::from_u32(code)? + } + 'u' => { + if chars.next()? != '{' { + return None; + } + let mut code = 0u32; + let mut digits = 0; + loop { + expired(deadline)?; + match chars.next()? { + '}' if digits > 0 => break, + '_' if digits > 0 => {} + ch => { + digits += 1; + if digits > 6 { + return None; + } + code = code.checked_mul(16)?.checked_add(ch.to_digit(16)?)?; + } + } + } + char::from_u32(code)? + } + '\n' => { + while chars + .peek() + .is_some_and(|ch| matches!(ch, ' ' | '\t' | '\r' | '\n')) + { + expired(deadline)?; + chars.next(); + } + continue; + } + _ => return None, + }); + } + Some(value) +} From 1d229b169cb1092b9682bb4fa9f7dd60430b7721 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:16:29 +0100 Subject: [PATCH 011/163] mason: recognize Rust macro matcher binding nodes (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 545adb87acde462e497cd6b37dd8465b3c5b28c4) (cherry picked from commit 3aff3d7f00357b32aaa7e8d472502a2c2f7bb260) --- .../src/lsp/completed_rust_check/inputs.rs | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/aft/src/lsp/completed_rust_check/inputs.rs b/crates/aft/src/lsp/completed_rust_check/inputs.rs index 7ae6804e49..d9cdaaab53 100644 --- a/crates/aft/src/lsp/completed_rust_check/inputs.rs +++ b/crates/aft/src/lsp/completed_rust_check/inputs.rs @@ -55,7 +55,7 @@ fn tokens(node: tree_sitter::Node<'_>, bytes: &[u8], deadline: Instant) -> Optio "token_tree" | "token_tree_pattern" => { result.push(Token::Group(tokens(child, bytes, deadline)?)) } - "token_binding" => result.extend(tokens(child, bytes, deadline)?), + "token_binding_pattern" => result.extend(tokens(child, bytes, deadline)?), _ => result.push(Token::Text(text.into())), } } @@ -557,3 +557,33 @@ fn string_literal(text: &str, deadline: Instant) -> Option { } Some(value) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn literal_macro_matcher_binds_its_metavariable() { + let bytes = br#"macro_rules! fixture { ($fixture:literal) => { include_str!(concat!("data/", $fixture)) }; }"#; + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&tree_sitter_rust::LANGUAGE.into()) + .unwrap(); + let tree = parser.parse(bytes, None).unwrap(); + let definition = tree.root_node().named_child(0).unwrap(); + let mut cursor = definition.walk(); + let rule = definition + .named_children(&mut cursor) + .find(|node| node.kind() == "macro_rule") + .unwrap(); + let left = rule.child_by_field_name("left").unwrap(); + let deadline = Instant::now() + std::time::Duration::from_secs(2); + let tokens = tokens(left, bytes, deadline).unwrap(); + assert_eq!( + parameters(&tokens, deadline), + Some(vec!["$fixture".into()]), + "{}: {tokens:?}", + left.to_sexp() + ); + } +} From f7c6b16bf8db35b23a2b5a419e4159506ae23445 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:33:20 +0100 Subject: [PATCH 012/163] mason: refuse hidden macro forwarding and generation (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 0aee1a46ac6f2bd0da50a562c2d0d2a3622f7722) (cherry picked from commit fedaf6ec7775e2321f1b29b6581de9f9d93f6a2e) --- crates/aft/src/lsp/completed_rust_check.rs | 3 +++ .../src/lsp/completed_rust_check/inputs.rs | 21 +++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/lsp/completed_rust_check.rs b/crates/aft/src/lsp/completed_rust_check.rs index dad0c4d293..ddc72b81c5 100644 --- a/crates/aft/src/lsp/completed_rust_check.rs +++ b/crates/aft/src/lsp/completed_rust_check.rs @@ -2627,6 +2627,7 @@ mod tests { "fixture!(concat!(\"a\", \".txt\"))", "forward!(fixture!(\"a.txt\"))", "macro_rules! forward { () => { fixture!(\"a.txt\") }; }", + "macro_rules! forward { ($($name:literal),*) => { $(fixture!($name);)* }; } forward!(\"a.txt\");", "forward!(fixture)", "use fixture as alias; alias!(\"a.txt\")", ] { @@ -2693,8 +2694,10 @@ mod tests { for definition in [ r#"macro_rules! fixture { ($($fixture:literal),*) => { include_str!(concat!("data/", $($fixture),*)) }; }"#, r#"macro_rules! fixture { ($fixture:literal) => { include_str!(concat!("data/", $fixture)) }; () => { "empty" }; }"#, + r#"macro_rules! fixture { ($fixture:literal) => { macro_rules! generated { () => { include_str!(concat!("data/", $fixture)) }; } generated!() }; }"#, ] { let (_temp, cache) = fixture(); + write(&cache.root, "member/src/data/a.txt", "input"); write( &cache.root, "member/src/lib.rs", diff --git a/crates/aft/src/lsp/completed_rust_check/inputs.rs b/crates/aft/src/lsp/completed_rust_check/inputs.rs index d9cdaaab53..0a0455bd5a 100644 --- a/crates/aft/src/lsp/completed_rust_check/inputs.rs +++ b/crates/aft/src/lsp/completed_rust_check/inputs.rs @@ -56,6 +56,9 @@ fn tokens(node: tree_sitter::Node<'_>, bytes: &[u8], deadline: Instant) -> Optio result.push(Token::Group(tokens(child, bytes, deadline)?)) } "token_binding_pattern" => result.extend(tokens(child, bytes, deadline)?), + _ if child.named_child_count() > 0 && child.kind() != "scoped_identifier" => { + result.push(Token::Group(tokens(child, bytes, deadline)?)); + } _ => result.push(Token::Text(text.into())), } } @@ -307,6 +310,13 @@ impl SourceInputs { let mut index = 0; while index < tokens.len() { expired(deadline)?; + if tokens[index].text("macro_rules") { + if let Some(definition) = definition { + // A generated macro has its own bindings and expansion + // sites; substituting the outer bindings would be a guess. + self.definitions[definition].parameters = None; + } + } if let [Token::Text(name), bang, Token::Group(args), ..] = &tokens[index..] { if bang.text("!") { let name = name.rsplit("::").next()?; @@ -341,7 +351,10 @@ impl SourceInputs { } included.push(target); } - None => return Self::refuse(path, false, unresolved), + None => { + expired(deadline)?; + return Self::refuse(path, false, unresolved); + } } } } @@ -416,6 +429,7 @@ impl SourceInputs { let Some(args) = arguments(&invocation.args, deadline) .filter(|args| args.len() == parameters.len()) else { + expired(deadline)?; return Self::refuse(&definition.path, false, unresolved); }; let mut bindings = BTreeMap::new(); @@ -438,7 +452,10 @@ impl SourceInputs { } included.push(target); } - None => return Self::refuse(&definition.path, false, unresolved), + None => { + expired(deadline)?; + return Self::refuse(&definition.path, false, unresolved); + } } } } From b672676ee37383309e16021d8611bb9798f43dc4 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:00:52 +0100 Subject: [PATCH 013/163] mason: cache Rust source facts and prefilter parsing (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 07c7cd0b092dafdf71e245aa1800446c1e228f6d) (cherry picked from commit 72241a06d6f0ebe80e42d478932b66d4d8d74d8d) --- crates/aft/examples/rust_fingerprint_probe.rs | 49 ++++ crates/aft/src/lsp/completed_rust_check.rs | 234 ++++++++++++++-- .../src/lsp/completed_rust_check/inputs.rs | 249 ++++++++++++++++-- crates/aft/src/lsp/mod.rs | 2 + 4 files changed, 485 insertions(+), 49 deletions(-) create mode 100644 crates/aft/examples/rust_fingerprint_probe.rs diff --git a/crates/aft/examples/rust_fingerprint_probe.rs b/crates/aft/examples/rust_fingerprint_probe.rs new file mode 100644 index 0000000000..5286ab54be --- /dev/null +++ b/crates/aft/examples/rust_fingerprint_probe.rs @@ -0,0 +1,49 @@ +//! Release-only wall/thread-CPU probe for the real Rust input fingerprint. +//! Run with `cargo run --release -p agent-file-tools --example rust_fingerprint_probe --features test-hooks -- `. + +#[cfg(all(feature = "test-hooks", target_os = "linux"))] +fn main() { + use std::time::{Duration, Instant}; + + use aft::lsp::FingerprintProbe; + + fn cpu() -> Duration { + let mut time = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // The pointer refers to writable storage with the layout clock_gettime expects. + assert_eq!( + unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut time) }, + 0 + ); + Duration::new(time.tv_sec as u64, time.tv_nsec as u32) + } + + let root = + std::fs::canonicalize(std::env::args_os().nth(1).expect("corpus path required")).unwrap(); + let start_cpu = cpu(); + let start = Instant::now(); + let cold = FingerprintProbe::capture(&root, None).expect("cold fingerprint must complete"); + let cold_wall_ms = start.elapsed().as_secs_f64() * 1000.0; + let cold_cpu_ms = (cpu() - start_cpu).as_secs_f64() * 1000.0; + let start_cpu = cpu(); + let start = Instant::now(); + let warm = + FingerprintProbe::capture(&root, Some(&cold)).expect("warm fingerprint must complete"); + let warm_wall_ms = start.elapsed().as_secs_f64() * 1000.0; + let warm_cpu_ms = (cpu() - start_cpu).as_secs_f64() * 1000.0; + assert!(cold.same_digest(&warm)); + let (examined, inputs, rust_files, rust_bytes, cold_hashed) = cold.stats(); + let (_, _, _, _, warm_hashed) = warm.stats(); + println!( + "{}", + serde_json::json!({"cold_wall_ms": cold_wall_ms, "cold_cpu_ms": cold_cpu_ms, "warm_wall_ms": warm_wall_ms, "warm_cpu_ms": warm_cpu_ms, "examined": examined, "inputs": inputs, "rust_files": rust_files, "rust_bytes": rust_bytes, "cold_hashed": cold_hashed, "warm_hashed": warm_hashed}) + ); +} + +#[cfg(not(all(feature = "test-hooks", target_os = "linux")))] +fn main() { + eprintln!("rust_fingerprint_probe requires Linux and --features test-hooks"); + std::process::exit(1); +} diff --git a/crates/aft/src/lsp/completed_rust_check.rs b/crates/aft/src/lsp/completed_rust_check.rs index ddc72b81c5..8ebe754c37 100644 --- a/crates/aft/src/lsp/completed_rust_check.rs +++ b/crates/aft/src/lsp/completed_rust_check.rs @@ -1,7 +1,7 @@ //! A completed compiler check, not an idle server, is the authority behind this //! cache. Disk validation is fail-closed and never installs cached rows into the //! live diagnostics store: a cold server can publish partial results independently. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs; use std::io::Read; use std::path::{Path, PathBuf}; @@ -14,11 +14,11 @@ use serde::{Deserialize, Serialize}; use super::diagnostics::{DiagnosticSeverity, StoredDiagnostic}; mod inputs; -use inputs::{SourceInputs, UnresolvedInput}; +use inputs::{CachedFacts, SourceInputs, UnresolvedInput}; pub(crate) const BUDGET: Duration = Duration::from_secs(2); // The first completed check has no trusted hash ledger yet. Later requests -// reuse content hashes, but still resolve macros against current source files. +// reuse content hashes and parsed facts, then resolve cross-file macro inputs. const INITIAL_CAPTURE_BUDGET: Duration = Duration::from_secs(15); // Filesystem timestamps can round a write down to a tick before the wall-clock // instant at which it actually happened. Two seconds covers coarse timestamp @@ -29,9 +29,9 @@ const CHECK_BEGIN_TIMESTAMP_MARGIN: Duration = Duration::from_secs(2); // Worktree churn must not turn completed compiler checks into an unbounded disk cache. const MAX_RECORDS: usize = 64; const RECORD_SCAN_LIMIT: usize = 1024; -// Schema 2 did not fence filesystem timestamp granularity at check begin, so -// even matching inputs cannot establish that its saved diagnostics are sound. -const RECORD_SCHEMA: u32 = 3; +// Older records have no proof that skipped sources were checked for the set of +// include-bearing macro names. Rebuild their facts before trusting a check. +const RECORD_SCHEMA: u32 = 4; fn canonical(path: &Path) -> Option { fs::canonicalize(path) @@ -150,6 +150,10 @@ pub(crate) struct Fingerprint { pub(crate) hashed: usize, extra_inputs: Vec, extra_env_keys: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + source_facts: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + macro_names: BTreeSet, #[serde(skip)] environment_overrides: BTreeMap, } @@ -250,7 +254,7 @@ fn workspace_packages(root: &Path, deadline: Instant) -> Option> { Some(packages) } -#[cfg(test)] +#[cfg(any(test, feature = "test-hooks"))] pub(crate) fn fingerprint( root: &Path, runtime: String, @@ -263,6 +267,46 @@ pub(crate) fn fingerprint( fingerprint_with_reason(root, runtime, previous, deadline, began, &mut None) } +/// Test-only access to the production fingerprint for paired release probes. +#[cfg(feature = "test-hooks")] +pub struct FingerprintProbe(Fingerprint); + +#[cfg(feature = "test-hooks")] +impl FingerprintProbe { + pub fn capture(root: &Path, previous: Option<&Self>) -> Option { + fingerprint( + root, + "release-probe".into(), + previous.map(|previous| &previous.0), + Instant::now() + Duration::from_secs(60), + None, + ) + .map(Self) + } + + pub fn stats(&self) -> (usize, usize, usize, u64, usize) { + let (rust_files, rust_bytes) = self + .0 + .files + .iter() + .filter(|(path, _)| path.extension().is_some_and(|ext| ext == "rs")) + .fold((0, 0), |(count, bytes), (_, stamp)| { + (count + 1, bytes + stamp.size) + }); + ( + self.0.examined, + self.0.files.len(), + rust_files, + rust_bytes, + self.0.hashed, + ) + } + + pub fn same_digest(&self, other: &Self) -> bool { + self.0.digest == other.0.digest + } +} + fn fingerprint_with_reason( root: &Path, runtime: String, @@ -284,7 +328,8 @@ fn fingerprint_with_reason( started: Instant::now(), }; let mut hashed = 0; - let mut source_inputs = SourceInputs::default(); + let mut source_inputs = + SourceInputs::new(previous.map_or_else(BTreeSet::new, |p| p.macro_names.clone())); let mut pending = workspace_packages(root, deadline)?; let mut inputs = vec![root.join("Cargo.toml")]; let extra_inputs = previous.map_or_else(Vec::new, |p| p.extra_inputs.clone()); @@ -374,9 +419,13 @@ fn fingerprint_with_reason( } loop { if inputs.is_empty() { + source_inputs.scan_pending(root, &files, deadline, unresolved)?; for path in source_inputs.resolve(root, deadline, unresolved)? { expired(deadline)?; - if !files.contains_key(&path) || source_inputs.needs_parse(&path) { + if !files.contains_key(&path) + || (!path.extension().is_some_and(|ext| ext == "rs") + && source_inputs.needs_parse(&path)) + { inputs.push(path); } } @@ -386,7 +435,10 @@ fn fingerprint_with_reason( }; report.last = path.clone(); expired(deadline)?; - if files.contains_key(&path) && !source_inputs.needs_parse(&path) { + if files.contains_key(&path) + && (path.extension().is_some_and(|ext| ext == "rs") + || !source_inputs.needs_parse(&path)) + { continue; } let metadata = fs::symlink_metadata(&path).ok()?; @@ -452,17 +504,14 @@ fn fingerprint_with_reason( hashed += 1; } if source_inputs.needs_parse(&path) { - // Macro invocations in another source file can change the paths a - // definition reads, even when the definition's own bytes did not. - let bytes = source_bytes.or_else(|| read_bounded(&path, deadline))?; - let observed = stamp(&fs::metadata(&path).ok()?)?; - if (observed.size, observed.modified, observed.changed) - != (current.size, current.modified, current.changed) - { - return None; - } + // Facts are keyed by the same content hash as the file stamp. + // Unchanged sources need neither a read nor a tree-sitter parse. + let cached = previous + .filter(|_| old.is_some_and(|old| old.hash == current.hash)) + .and_then(|previous| previous.source_facts.get(&path)) + .cloned(); let (included, env_keys) = - source_inputs.add(root, &path, &bytes, deadline, unresolved)?; + source_inputs.add(root, &path, source_bytes, cached, deadline, unresolved)?; current.included = included; current.env_keys = env_keys; } @@ -502,6 +551,7 @@ fn fingerprint_with_reason( hashed ); report.complete = true; + let macro_names = source_inputs.macro_names(); Some(Fingerprint { files, runtime, @@ -510,6 +560,8 @@ fn fingerprint_with_reason( hashed, extra_inputs, extra_env_keys, + source_facts: source_inputs.facts, + macro_names, environment_overrides, }) } @@ -687,6 +739,8 @@ impl Runtime { hashed: 0, extra_inputs: Vec::new(), extra_env_keys: Vec::new(), + source_facts: BTreeMap::new(), + macro_names: BTreeSet::new(), environment_overrides: BTreeMap::new(), }); seed.environment_overrides = self @@ -930,8 +984,16 @@ fn map_fingerprint_paths( fingerprint.extra_inputs = fingerprint .extra_inputs .into_iter() - .map(map) + .map(&mut map) .collect::>()?; + let mut source_facts = BTreeMap::new(); + for (path, mut facts) in fingerprint.source_facts { + facts.map_paths(&mut map)?; + if source_facts.insert(map(path)?, facts).is_some() { + return Err("duplicate source facts path"); + } + } + fingerprint.source_facts = source_facts; Ok(fingerprint) } @@ -2316,10 +2378,22 @@ mod tests { write_record(&missed.path, certified).unwrap(); let disk: serde_json::Value = serde_json::from_slice(&fs::read(&missed.path).unwrap()).unwrap(); - assert_eq!(disk["record"]["schema"], 3); + assert_eq!(disk["record"]["schema"], RECORD_SCHEMA); assert!(valid(&reload(&missed)).is_some()); } + #[test] + fn schema_three_record_without_source_facts_is_a_cache_miss() { + let (_temp, cache) = fixture(); + let mut old = (**cache.saved.as_ref().unwrap()).clone(); + old.schema = 3; + write_record(&cache.path, &old).unwrap(); + assert!( + reload(&cache).saved.is_none(), + "older records do not prove which macro names skipped sources were checked against" + ); + } + #[test] fn fingerprint_deadline_exceeded_refuses_saved_check() { let (_temp, cache) = fixture(); @@ -2429,6 +2503,122 @@ mod tests { ) } + #[test] + fn warm_fingerprint_reuses_saved_source_facts_without_parsing() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "member/src/data/a.txt", "one"); + write(&cache.root, "member/src/data/b.txt", "two"); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { ($name:literal) => { include_str!(concat!("data/", $name)) }; } + const ENV: Option<&str> = option_env!("AFT_TEST_CACHED_SOURCE_FACTS"); + "#, + ); + write( + &cache.root, + "member/src/calls.rs", + "const DATA: &str = fixture!(\"a.txt\");", + ); + inputs::take_parses(); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert_eq!( + inputs::take_parses(), + 2, + "only the definition and its call need parsing" + ); + write_record(&cache.path, cache.saved.as_ref().unwrap()).unwrap(); + let reloaded = reload(&cache); + let original = &cache.saved.as_ref().unwrap().fingerprint; + let restored = &reloaded.saved.as_ref().unwrap().fingerprint; + assert_eq!( + serde_json::to_value(&original.source_facts).unwrap(), + serde_json::to_value(&restored.source_facts).unwrap() + ); + assert_eq!(original.macro_names, restored.macro_names); + inputs::take_parses(); + let warm = fingerprint( + &cache.root, + "v1".into(), + Some(restored), + Instant::now() + BUDGET, + None, + ) + .unwrap(); + assert_eq!( + inputs::take_parses(), + 0, + "unchanged warm sources must not be parsed" + ); + assert_eq!(warm.digest, restored.digest); + write( + &cache.root, + "member/src/calls.rs", + "const DATA: &str = fixture!(\"b.txt\");", + ); + let changed = fingerprint( + &cache.root, + "v1".into(), + Some(&warm), + Instant::now() + BUDGET, + None, + ) + .unwrap(); + assert_eq!( + inputs::take_parses(), + 1, + "only the changed call site needs parsing" + ); + assert!(changed + .files + .contains_key(&cache.root.join("member/src/data/b.txt"))); + assert!(!changed + .files + .contains_key(&cache.root.join("member/src/data/a.txt"))); + } + + #[test] + fn byte_prefilter_skips_irrelevant_sources_and_revisits_new_macro_names() { + let (_temp, cache) = fixture(); + write( + &cache.root, + "member/src/calls.rs", + "const DATA: &str = fixture!(\"a.txt\");", + ); + write(&cache.root, "member/src/data/a.txt", "one"); + inputs::take_parses(); + let before = capture_inputs(&cache).unwrap(); + assert_eq!( + inputs::take_parses(), + 0, + "irrelevant bytes must not reach tree-sitter" + ); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { ($name:literal) => { include_str!(concat!("data/", $name)) }; } + "#, + ); + let after = fingerprint( + &cache.root, + "v1".into(), + Some(&before), + Instant::now() + BUDGET, + None, + ) + .unwrap(); + assert_eq!( + inputs::take_parses(), + 2, + "a new definition must discover unchanged callers" + ); + assert!(after + .files + .contains_key(&cache.root.join("member/src/data/a.txt"))); + } + fn unresolved_input(cache: &CompletedRustCheck) -> String { let mut unresolved = None; assert!(fingerprint_with_reason( diff --git a/crates/aft/src/lsp/completed_rust_check/inputs.rs b/crates/aft/src/lsp/completed_rust_check/inputs.rs index 0a0455bd5a..38f8f3af07 100644 --- a/crates/aft/src/lsp/completed_rust_check/inputs.rs +++ b/crates/aft/src/lsp/completed_rust_check/inputs.rs @@ -2,9 +2,11 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::Instant; use super::{canonical, expired}; +use serde::{Deserialize, Serialize}; const INCLUDE_PROVISIONAL: &str = "cargo check completed, but {file} includes a file whose path can't be resolved; diagnostics are provisional"; const ENV_PROVISIONAL: &str = "cargo check completed, but {file} reads an environment variable whose name can't be resolved; diagnostics are provisional"; @@ -26,7 +28,7 @@ impl UnresolvedInput { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] enum Token { Literal(String), Text(String), @@ -141,6 +143,7 @@ fn include_path(root: &Path, source: &Path, relative: &str, deadline: Instant) - (path.starts_with(root) && path.is_file()).then_some(path) } +#[derive(Clone, Debug, Serialize, Deserialize)] struct Definition { name: String, path: PathBuf, @@ -148,6 +151,7 @@ struct Definition { includes: Vec<(Vec, bool)>, } +#[derive(Clone, Debug, Serialize, Deserialize)] struct Invocation { name: String, path: PathBuf, @@ -155,22 +159,56 @@ struct Invocation { nested: bool, } -#[derive(Default)] -pub(super) struct SourceInputs { +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct FileFacts { definitions: Vec, invocations: Vec, - references: Vec, - rust_sources: BTreeSet, - parsed: BTreeSet, + references: BTreeSet, + direct: Vec<(Vec, bool)>, + env_keys: Vec, } -impl SourceInputs { - pub(super) fn needs_parse(&self, path: &Path) -> bool { - !self.parsed.contains(path) - && (path.extension().is_some_and(|extension| extension == "rs") - || self.rust_sources.contains(path)) +#[derive(Clone, Debug)] +pub(super) struct CachedFacts(Arc); + +impl Serialize for CachedFacts { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for CachedFacts { + fn deserialize>(deserializer: D) -> Result { + FileFacts::deserialize(deserializer).map(|facts| Self(Arc::new(facts))) + } +} + +impl CachedFacts { + pub(super) fn map_paths( + &mut self, + map: &mut impl FnMut(PathBuf) -> Result, + ) -> Result<(), &'static str> { + let facts = Arc::make_mut(&mut self.0); + for definition in &mut facts.definitions { + definition.path = map(definition.path.clone())?; + } + for invocation in &mut facts.invocations { + invocation.path = map(invocation.path.clone())?; + } + Ok(()) } +} +#[derive(Default)] +pub(super) struct SourceInputs { + pub(super) facts: BTreeMap, + rust_sources: BTreeSet, + parsed: BTreeSet, + pending: BTreeMap>>, + searched_names: BTreeSet, +} + +impl FileFacts { fn refuse( path: &Path, environment: bool, @@ -185,7 +223,28 @@ impl SourceInputs { } impl SourceInputs { - pub(super) fn add( + pub(super) fn new(searched_names: BTreeSet) -> Self { + Self { + searched_names, + ..Self::default() + } + } + pub(super) fn needs_parse(&self, path: &Path) -> bool { + path.extension().is_some_and(|extension| extension == "rs") + || (!self.parsed.contains(path) && self.rust_sources.contains(path)) + } +} + +#[cfg(test)] +thread_local! { static PARSES: std::cell::Cell = const { std::cell::Cell::new(0) }; } + +#[cfg(test)] +pub(super) fn take_parses() -> usize { + PARSES.with(|count| count.replace(0)) +} + +impl FileFacts { + fn parse( &mut self, root: &Path, path: &Path, @@ -194,7 +253,8 @@ impl SourceInputs { unresolved: &mut Option, ) -> Option<(Vec, Vec)> { std::str::from_utf8(bytes).ok()?; - self.parsed.insert(path.into()); + #[cfg(test)] + PARSES.with(|count| count.set(count.get() + 1)); let mut parser = tree_sitter::Parser::new(); parser .set_language(&tree_sitter_rust::LANGUAGE.into()) @@ -283,7 +343,7 @@ impl SourceInputs { continue; } if node.kind() == "identifier" { - self.references.push(node.utf8_text(bytes).ok()?.into()); + self.references.insert(node.utf8_text(bytes).ok()?.into()); } let mut cursor = node.walk(); for child in node.named_children(&mut cursor) { @@ -291,6 +351,7 @@ impl SourceInputs { nodes.push(child); } } + self.env_keys = env_keys.clone(); Some((included, env_keys)) } @@ -339,6 +400,7 @@ impl SourceInputs { .includes .push((args.to_vec(), name == "include")); } else { + self.direct.push((args.to_vec(), name == "include")); let empty = BTreeMap::new(); let target = expression(args, path, &empty, deadline).and_then(|relative| { @@ -346,9 +408,6 @@ impl SourceInputs { }); match target { Some(target) => { - if name == "include" { - self.rust_sources.insert(target.clone()); - } included.push(target); } None => { @@ -387,12 +446,148 @@ impl SourceInputs { root, path, group, definition, nested, deadline, included, env_keys, unresolved, )?; } else if let Token::Text(name) = &tokens[index] { - self.references.push(name.clone()); + self.references.insert(name.rsplit("::").next()?.into()); } index += 1; } Some(()) } +} + +impl SourceInputs { + pub(super) fn macro_names(&self) -> BTreeSet { + self.facts + .values() + .flat_map(|facts| &facts.0.definitions) + .filter(|definition| !definition.includes.is_empty()) + .map(|definition| definition.name.clone()) + .collect() + } + + pub(super) fn add( + &mut self, + root: &Path, + path: &Path, + bytes: Option>, + cached: Option, + deadline: Instant, + unresolved: &mut Option, + ) -> Option<(Vec, Vec)> { + if !path.extension().is_some_and(|extension| extension == "rs") { + self.parsed.insert(path.into()); + } + if let Some(cached) = cached { + return self.load(root, path, cached, deadline, unresolved); + } + if let Some(bytes) = bytes { + let text = std::str::from_utf8(&bytes).ok()?; + if text.contains("include") || text.contains("env") { + let mut facts = FileFacts::default(); + facts.parse(root, path, &bytes, deadline, unresolved)?; + return self.load( + root, + path, + CachedFacts(Arc::new(facts)), + deadline, + unresolved, + ); + } + self.pending.insert(path.into(), Some(bytes)); + } + Some((Vec::new(), Vec::new())) + } + + fn load( + &mut self, + root: &Path, + path: &Path, + facts: CachedFacts, + deadline: Instant, + unresolved: &mut Option, + ) -> Option<(Vec, Vec)> { + let mut included = Vec::new(); + for (input, rust) in &facts.0.direct { + expired(deadline)?; + let target = expression(input, path, &BTreeMap::new(), deadline) + .and_then(|relative| include_path(root, path, &relative, deadline)); + let Some(target) = target else { + expired(deadline)?; + return FileFacts::refuse(path, false, unresolved); + }; + if *rust { + self.rust_sources.insert(target.clone()); + } + included.push(target); + } + let env_keys = facts.0.env_keys.clone(); + self.facts.insert(path.into(), facts); + Some((included, env_keys)) + } + + pub(super) fn scan_pending( + &mut self, + root: &Path, + files: &BTreeMap, + deadline: Instant, + unresolved: &mut Option, + ) -> Option<()> { + let names = self.macro_names(); + let added: BTreeSet<_> = names.difference(&self.searched_names).cloned().collect(); + if !added.is_empty() { + for path in files.keys() { + expired(deadline)?; + if (path.extension().is_some_and(|ext| ext == "rs") + || self.rust_sources.contains(path)) + && !self.facts.contains_key(path) + { + self.pending.entry(path.clone()).or_insert(None); + } + } + } + let pending = std::mem::take(&mut self.pending); + for (path, bytes) in pending { + expired(deadline)?; + let candidates = if bytes.is_some() { &names } else { &added }; + if candidates.is_empty() { + continue; + } + let bytes = bytes.or_else(|| super::read_bounded(&path, deadline))?; + let observed = super::stamp(&std::fs::metadata(&path).ok()?)?; + let expected = files.get(&path)?; + if (observed.size, observed.modified, observed.changed) + != (expected.size, expected.modified, expected.changed) + { + return None; + } + let text = std::str::from_utf8(&bytes).ok()?; + let mut relevant = false; + for name in candidates { + expired(deadline)?; + // Also match bare names: an import alias or forwarding macro + // can hide a call without spelling `name!` in the same file. + if text.contains(name) { + relevant = true; + break; + } + } + if relevant { + if blake3::hash(&bytes).to_hex().as_str() != expected.hash { + return None; + } + let mut facts = FileFacts::default(); + facts.parse(root, &path, &bytes, deadline, unresolved)?; + self.load( + root, + &path, + CachedFacts(Arc::new(facts)), + deadline, + unresolved, + )?; + } + } + self.searched_names.extend(names); + Some(()) + } pub(super) fn resolve( &mut self, @@ -401,20 +596,20 @@ impl SourceInputs { unresolved: &mut Option, ) -> Option> { let mut included = Vec::new(); - for definition in &self.definitions { + for definition in self.facts.values().flat_map(|facts| &facts.0.definitions) { expired(deadline)?; if definition.includes.is_empty() { continue; } - for name in &self.references { + for facts in self.facts.values() { expired(deadline)?; // Passing a macro by name (including a `use ... as ...` // alias) can hide calls from this deliberately small resolver. - if name.rsplit("::").next() == Some(&definition.name) { - return Self::refuse(&definition.path, false, unresolved); + if facts.0.references.contains(&definition.name) { + return FileFacts::refuse(&definition.path, false, unresolved); } } - for invocation in &self.invocations { + for invocation in self.facts.values().flat_map(|facts| &facts.0.invocations) { expired(deadline)?; if invocation.name != definition.name { continue; @@ -424,19 +619,19 @@ impl SourceInputs { .as_ref() .filter(|_| !invocation.nested) else { - return Self::refuse(&definition.path, false, unresolved); + return FileFacts::refuse(&definition.path, false, unresolved); }; let Some(args) = arguments(&invocation.args, deadline) .filter(|args| args.len() == parameters.len()) else { expired(deadline)?; - return Self::refuse(&definition.path, false, unresolved); + return FileFacts::refuse(&definition.path, false, unresolved); }; let mut bindings = BTreeMap::new(); for (parameter, arg) in parameters.iter().zip(args) { expired(deadline)?; let [Token::Literal(value)] = arg else { - return Self::refuse(&definition.path, false, unresolved); + return FileFacts::refuse(&definition.path, false, unresolved); }; bindings.insert(parameter.clone(), value.clone()); } @@ -454,7 +649,7 @@ impl SourceInputs { } None => { expired(deadline)?; - return Self::refuse(&definition.path, false, unresolved); + return FileFacts::refuse(&definition.path, false, unresolved); } } } diff --git a/crates/aft/src/lsp/mod.rs b/crates/aft/src/lsp/mod.rs index 7beafbebcd..173d1755bc 100644 --- a/crates/aft/src/lsp/mod.rs +++ b/crates/aft/src/lsp/mod.rs @@ -2,6 +2,8 @@ mod applicability_cache; pub mod child_registry; pub mod client; pub(crate) mod completed_rust_check; +#[cfg(feature = "test-hooks")] +pub use completed_rust_check::FingerprintProbe; mod diagnostic_registration; pub mod diagnostics; pub mod document; From 2b5844bc8295c5da8710aaee9d69b1677fa3b3b4 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:31:24 +0100 Subject: [PATCH 014/163] mason: expose failed fingerprint trace to release probes (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 363ba9027a6f13e64de9da923e656654f06e83cf) (cherry picked from commit 423fce74ed7ed6cc216ac8414e4e9723d0581ac7) --- crates/aft/src/lsp/completed_rust_check.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/aft/src/lsp/completed_rust_check.rs b/crates/aft/src/lsp/completed_rust_check.rs index 8ebe754c37..52db367838 100644 --- a/crates/aft/src/lsp/completed_rust_check.rs +++ b/crates/aft/src/lsp/completed_rust_check.rs @@ -56,7 +56,7 @@ impl Drop for WalkReport<'_> { self.last.display(), self.started.elapsed().as_millis() ); - #[cfg(test)] + #[cfg(any(test, feature = "test-hooks"))] if !self.complete { eprintln!( "fingerprint refused at {} after {:?}; examined {}", From 3a1f655deba5b18562dfb89a534a8dde57c70f14 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:41:24 +0100 Subject: [PATCH 015/163] mason: short-circuit unsupported local include macro calls (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit b4a8eada0f9c68abd1d996ba1e0c4852dcd934f4) (cherry picked from commit cfaeeee11cf5a4dee2fe10de8189eba39775d281) --- crates/aft/examples/rust_fingerprint_probe.rs | 11 ++++-- .../src/lsp/completed_rust_check/inputs.rs | 36 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/crates/aft/examples/rust_fingerprint_probe.rs b/crates/aft/examples/rust_fingerprint_probe.rs index 5286ab54be..86cef81897 100644 --- a/crates/aft/examples/rust_fingerprint_probe.rs +++ b/crates/aft/examples/rust_fingerprint_probe.rs @@ -24,9 +24,16 @@ fn main() { std::fs::canonicalize(std::env::args_os().nth(1).expect("corpus path required")).unwrap(); let start_cpu = cpu(); let start = Instant::now(); - let cold = FingerprintProbe::capture(&root, None).expect("cold fingerprint must complete"); + let cold = FingerprintProbe::capture(&root, None); let cold_wall_ms = start.elapsed().as_secs_f64() * 1000.0; let cold_cpu_ms = (cpu() - start_cpu).as_secs_f64() * 1000.0; + let Some(cold) = cold else { + println!( + "{}", + serde_json::json!({"complete": false, "cold_wall_ms": cold_wall_ms, "cold_cpu_ms": cold_cpu_ms}) + ); + return; + }; let start_cpu = cpu(); let start = Instant::now(); let warm = @@ -38,7 +45,7 @@ fn main() { let (_, _, _, _, warm_hashed) = warm.stats(); println!( "{}", - serde_json::json!({"cold_wall_ms": cold_wall_ms, "cold_cpu_ms": cold_cpu_ms, "warm_wall_ms": warm_wall_ms, "warm_cpu_ms": warm_cpu_ms, "examined": examined, "inputs": inputs, "rust_files": rust_files, "rust_bytes": rust_bytes, "cold_hashed": cold_hashed, "warm_hashed": warm_hashed}) + serde_json::json!({"complete": true, "cold_wall_ms": cold_wall_ms, "cold_cpu_ms": cold_cpu_ms, "warm_wall_ms": warm_wall_ms, "warm_cpu_ms": warm_cpu_ms, "examined": examined, "inputs": inputs, "rust_files": rust_files, "rust_bytes": rust_bytes, "cold_hashed": cold_hashed, "warm_hashed": warm_hashed}) ); } diff --git a/crates/aft/src/lsp/completed_rust_check/inputs.rs b/crates/aft/src/lsp/completed_rust_check/inputs.rs index 38f8f3af07..29bdf1e64f 100644 --- a/crates/aft/src/lsp/completed_rust_check/inputs.rs +++ b/crates/aft/src/lsp/completed_rust_check/inputs.rs @@ -352,6 +352,23 @@ impl FileFacts { } } self.env_keys = env_keys.clone(); + // An include-bearing macro invoked in this file with an unsupported + // matcher or a nested call is already uncertain. More files cannot + // resolve that call, so refuse before hashing the rest of the tree. + for definition in &self.definitions { + expired(deadline)?; + if definition.includes.is_empty() { + continue; + } + for invocation in &self.invocations { + expired(deadline)?; + if invocation.name == definition.name + && (definition.parameters.is_none() || invocation.nested) + { + return Self::refuse(&definition.path, false, unresolved); + } + } + } Some((included, env_keys)) } @@ -774,6 +791,25 @@ fn string_literal(text: &str, deadline: Instant) -> Option { mod tests { use super::*; + #[test] + fn unsupported_invoked_matcher_refuses_during_file_scan() { + let mut inputs = SourceInputs::default(); + let mut unresolved = None; + let bytes = br#"macro_rules! many { ($($name:literal),*) => { $(include_str!(concat!("data/", $name)))* }; } many!("a.txt");"#; + assert!(inputs + .add( + Path::new("workspace"), + Path::new("workspace/vectors.rs"), + Some(bytes.to_vec()), + None, + Instant::now() + std::time::Duration::from_secs(2), + &mut unresolved + ) + .is_none()); + assert!(unresolved.is_some()); + assert!(inputs.facts.is_empty()); + } + #[test] fn literal_macro_matcher_binds_its_metavariable() { let bytes = br#"macro_rules! fixture { ($fixture:literal) => { include_str!(concat!("data/", $fixture)) }; }"#; From 3a2cf7711af0d2c6b230b223b4a0ea2c935411d3 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:57:29 +0100 Subject: [PATCH 016/163] mason: keep unrelated byte and C strings out of path decoding (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 10bba5ac007d61177ffa7955c22ec2a012c3e9b4) (cherry picked from commit 3265bdbc897f4a5320d3d8861bb304b25f224ab8) --- crates/aft/src/lsp/completed_rust_check.rs | 18 ++++++++++++++++++ .../aft/src/lsp/completed_rust_check/inputs.rs | 9 +++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/lsp/completed_rust_check.rs b/crates/aft/src/lsp/completed_rust_check.rs index 52db367838..c0f6ee5b9a 100644 --- a/crates/aft/src/lsp/completed_rust_check.rs +++ b/crates/aft/src/lsp/completed_rust_check.rs @@ -2937,6 +2937,24 @@ mod tests { assert!(valid(&cache).is_none()); } + #[test] + fn unrelated_byte_and_c_string_macro_arguments_do_not_make_inputs_provisional() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "member/src/data.txt", "before"); + write( + &cache.root, + "member/src/lib.rs", + r##" + const DATA: &str = include_str!("data.txt"); + opaque!(b"\xff", br#"raw bytes"#, c"c string", cr#"raw c string"#); + "##, + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write(&cache.root, "member/src/data.txt", "after!"); + assert!(valid(&cache).is_none()); + } + #[test] fn declared_build_input_directory_addition_invalidates_completed_check() { let (_temp, mut cache) = fixture(); diff --git a/crates/aft/src/lsp/completed_rust_check/inputs.rs b/crates/aft/src/lsp/completed_rust_check/inputs.rs index 29bdf1e64f..9192bcdfa7 100644 --- a/crates/aft/src/lsp/completed_rust_check/inputs.rs +++ b/crates/aft/src/lsp/completed_rust_check/inputs.rs @@ -49,11 +49,16 @@ fn tokens(node: tree_sitter::Node<'_>, bytes: &[u8], deadline: Instant) -> Optio let text = child.utf8_text(bytes).ok()?; match child.kind() { "line_comment" | "block_comment" | "(" | ")" | "[" | "]" | "{" | "}" => {} - "string_literal" => result.push(Token::Literal(string_literal(text, deadline)?)), - "raw_string_literal" => result.push(Token::Literal( + "string_literal" => result.push( + string_literal(text, deadline) + .map(Token::Literal) + .unwrap_or_else(|| Token::Text(text.into())), + ), + "raw_string_literal" if text.starts_with('r') => result.push(Token::Literal( text.get(text.find('"')? + 1..text.rfind('"')?)? .replace("\r\n", "\n"), )), + "raw_string_literal" => result.push(Token::Text(text.into())), "token_tree" | "token_tree_pattern" => { result.push(Token::Group(tokens(child, bytes, deadline)?)) } From 1612b5c6f07f05b2ddb4720f7a88c2444a3d85cb Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:33:42 +0100 Subject: [PATCH 017/163] mason: print source uncertainty in fingerprint probes (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit d45a4ae8924f9caa6eddf03b23c2f0ad84b20703) (cherry picked from commit f6d02160b240090c80e1a4881bcfe38a364c4cee) --- crates/aft/src/lsp/completed_rust_check.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/aft/src/lsp/completed_rust_check.rs b/crates/aft/src/lsp/completed_rust_check.rs index c0f6ee5b9a..3bcb7fd885 100644 --- a/crates/aft/src/lsp/completed_rust_check.rs +++ b/crates/aft/src/lsp/completed_rust_check.rs @@ -264,7 +264,14 @@ pub(crate) fn fingerprint( // or directory may have changed since the reader observed that event. began: Option, ) -> Option { - fingerprint_with_reason(root, runtime, previous, deadline, began, &mut None) + let mut unresolved = None; + let captured = + fingerprint_with_reason(root, runtime, previous, deadline, began, &mut unresolved); + #[cfg(feature = "test-hooks")] + if let Some(unresolved) = unresolved { + eprintln!("{}", unresolved.note(root)); + } + captured } /// Test-only access to the production fingerprint for paired release probes. From bcef0ca2619c894e07464998143007cc8c51e898 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:55:06 +0100 Subject: [PATCH 018/163] mason: keep ordinary macro reexports resolvable (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit abf04aec2ad1de94013f4f5de85b19bb0f6a55c1) (cherry picked from commit afb4b4c1d49dcdda6ecbbf84a9d879c12122a099) --- crates/aft/src/lsp/completed_rust_check.rs | 26 +++++++++++++++++++ .../src/lsp/completed_rust_check/inputs.rs | 7 +++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/lsp/completed_rust_check.rs b/crates/aft/src/lsp/completed_rust_check.rs index 3bcb7fd885..6698136db8 100644 --- a/crates/aft/src/lsp/completed_rust_check.rs +++ b/crates/aft/src/lsp/completed_rust_check.rs @@ -2791,6 +2791,32 @@ mod tests { assert!(capture_inputs(&cache).is_some()); } + #[test] + fn macro_reexports_without_renaming_keep_exact_invocations() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "member/data/a.txt", "before"); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { ($name:literal) => { include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/", $name)) }; } + pub(crate) use fixture; + "#, + ); + write( + &cache.root, + "member/tests/check.rs", + r#" + use member::fixture; + const DATA: &str = member::fixture!("a.txt"); + "#, + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + write(&cache.root, "member/data/a.txt", "after!"); + assert!(valid(&cache).is_none()); + } + #[test] fn unresolved_include_names_its_source_with_approved_wording() { let (_temp, cache) = fixture(); diff --git a/crates/aft/src/lsp/completed_rust_check/inputs.rs b/crates/aft/src/lsp/completed_rust_check/inputs.rs index 9192bcdfa7..461d50ffe7 100644 --- a/crates/aft/src/lsp/completed_rust_check/inputs.rs +++ b/crates/aft/src/lsp/completed_rust_check/inputs.rs @@ -347,8 +347,11 @@ impl FileFacts { )?; continue; } - if node.kind() == "identifier" { - self.references.insert(node.utf8_text(bytes).ok()?.into()); + if node.kind() == "use_as_clause" { + // Renaming can hide an invocation's original macro name. + // Ordinary imports preserve the name used by the resolver. + let original = node.child_by_field_name("path")?.utf8_text(bytes).ok()?; + self.references.insert(original.rsplit("::").next()?.into()); } let mut cursor = node.walk(); for child in node.named_children(&mut cursor) { From 57361f896bb7dc8bd167cbabe0b85d95e2252f1e Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:32:55 +0100 Subject: [PATCH 019/163] mason: prefilter macro facts after discovering include names (#1425) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 764468d9efa179879d7c680c6d09d4427c6de818) (cherry picked from commit b44c619377370513f943d0c117cfc291f610409f) --- crates/aft/src/lsp/completed_rust_check.rs | 60 ++++++- .../src/lsp/completed_rust_check/inputs.rs | 147 +++++++++++++----- 2 files changed, 171 insertions(+), 36 deletions(-) diff --git a/crates/aft/src/lsp/completed_rust_check.rs b/crates/aft/src/lsp/completed_rust_check.rs index 6698136db8..dbf59d7eae 100644 --- a/crates/aft/src/lsp/completed_rust_check.rs +++ b/crates/aft/src/lsp/completed_rust_check.rs @@ -2626,6 +2626,39 @@ mod tests { .contains_key(&cache.root.join("member/src/data/a.txt"))); } + #[test] + fn new_macro_names_revisit_previously_cached_primary_files() { + let (_temp, cache) = fixture(); + write( + &cache.root, + "member/src/calls.rs", + "// env prefilter\nconst DATA: &str = fixture!(\"a.txt\");", + ); + write(&cache.root, "member/src/data/a.txt", "one"); + inputs::take_parses(); + let before = capture_inputs(&cache).unwrap(); + assert_eq!(inputs::take_parses(), 1); + write( + &cache.root, + "member/src/lib.rs", + r#" + macro_rules! fixture { ($name:literal) => { include_str!(concat!("data/", $name)) }; } + "#, + ); + let after = fingerprint( + &cache.root, + "v1".into(), + Some(&before), + Instant::now() + BUDGET, + None, + ) + .unwrap(); + assert_eq!(inputs::take_parses(), 2); + assert!(after + .files + .contains_key(&cache.root.join("member/src/data/a.txt"))); + } + fn unresolved_input(cache: &CompletedRustCheck) -> String { let mut unresolved = None; assert!(fingerprint_with_reason( @@ -2912,6 +2945,31 @@ mod tests { assert_eq!(unresolved_input(&cache), "cargo check completed, but member/src/lib.rs includes a file whose path can't be resolved; diagnostics are provisional"); } + #[cfg(unix)] + #[test] + fn warm_fingerprint_rechecks_excluded_include_parent_symlinks() { + let (_temp, mut cache) = fixture(); + write(&cache.root, "target/inputs/a.txt", "input"); + write( + &cache.root, + "member/src/lib.rs", + r#"const DATA: &str = include_str!(concat!("../../target/inputs/", "a.txt"));"#, + ); + cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); + assert!(valid(&cache).is_some()); + fs::rename( + cache.root.join("target/inputs"), + cache.root.join("target/actual"), + ) + .unwrap(); + std::os::unix::fs::symlink( + cache.root.join("target/actual"), + cache.root.join("target/inputs"), + ) + .unwrap(); + assert!(valid(&cache).is_none()); + } + #[test] fn complex_fixture_matchers_stay_provisional_when_invoked() { for definition in [ @@ -2979,7 +3037,7 @@ mod tests { "member/src/lib.rs", r##" const DATA: &str = include_str!("data.txt"); - opaque!(b"\xff", br#"raw bytes"#, c"c string", cr#"raw c string"#); + opaque!(b"\xff", br#"raw bytes"#, c"c string", cr#"raw c string"#, "include"); "##, ); cache.saved_mut().fingerprint = capture_inputs(&cache).unwrap(); diff --git a/crates/aft/src/lsp/completed_rust_check/inputs.rs b/crates/aft/src/lsp/completed_rust_check/inputs.rs index 461d50ffe7..7f6079afa4 100644 --- a/crates/aft/src/lsp/completed_rust_check/inputs.rs +++ b/crates/aft/src/lsp/completed_rust_check/inputs.rs @@ -1,5 +1,5 @@ //! Resolve compiler inputs only when the source proves every concrete path. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -134,18 +134,33 @@ fn expression( // Canonicalization alone would hide a symlink in an include's parent directory. // Check every component first, including components cancelled by `..`. -fn include_path(root: &Path, source: &Path, relative: &str, deadline: Instant) -> Option { +fn include_path( + root: &Path, + source: &Path, + relative: &str, + deadline: Instant, + checked: &mut HashSet, + resolved: &mut HashMap, +) -> Option { + expired(deadline)?; let joined = source.parent()?.join(relative); + if let Some(path) = resolved.get(&joined) { + return Some(path.clone()); + } let mut current = PathBuf::new(); for component in joined.components() { expired(deadline)?; current.push(component); - if fs::symlink_metadata(¤t).ok()?.is_symlink() { + if checked.insert(current.clone()) && fs::symlink_metadata(¤t).ok()?.is_symlink() { return None; } } let path = canonical(&joined)?; - (path.starts_with(root) && path.is_file()).then_some(path) + if !path.starts_with(root) || !path.is_file() { + return None; + } + resolved.insert(joined, path.clone()); + Some(path) } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -171,6 +186,8 @@ struct FileFacts { references: BTreeSet, direct: Vec<(Vec, bool)>, env_keys: Vec, + #[serde(default)] + scanned_names: BTreeSet, } #[derive(Clone, Debug)] @@ -211,6 +228,10 @@ pub(super) struct SourceInputs { parsed: BTreeSet, pending: BTreeMap>>, searched_names: BTreeSet, + // Repeated includes share checks only inside this capture, never across + // fingerprints where a parent directory could have become a symlink. + checked_components: HashSet, + resolved_paths: HashMap, } impl FileFacts { @@ -254,6 +275,7 @@ impl FileFacts { root: &Path, path: &Path, bytes: &[u8], + macro_names: &BTreeSet, deadline: Instant, unresolved: &mut Option, ) -> Option<(Vec, Vec)> { @@ -279,6 +301,7 @@ impl FileFacts { let mut nodes = vec![tree.root_node()]; let mut included = Vec::new(); let mut env_keys = Vec::new(); + let mut calls = Vec::new(); while let Some(node) = nodes.pop() { expired(deadline)?; if node.kind() == "macro_definition" { @@ -334,17 +357,7 @@ impl FileFacts { continue; } if node.kind() == "macro_invocation" { - self.scan( - root, - path, - &tokens(node, bytes, deadline)?, - None, - false, - deadline, - &mut included, - &mut env_keys, - unresolved, - )?; + calls.push(node); continue; } if node.kind() == "use_as_clause" { @@ -359,6 +372,42 @@ impl FileFacts { nodes.push(child); } } + // Templates reveal local include-bearing names before ordinary calls + // are tokenized. Most assert!/format! arguments are irrelevant inputs. + self.scanned_names = macro_names.clone(); + self.scanned_names.extend( + self.definitions + .iter() + .filter(|definition| !definition.includes.is_empty()) + .map(|definition| definition.name.clone()), + ); + for call in calls { + expired(deadline)?; + let text = call.utf8_text(bytes).ok()?; + let mut relevant = text.contains("include") || text.contains("env"); + if !relevant { + for name in &self.scanned_names { + expired(deadline)?; + if text.contains(name) { + relevant = true; + break; + } + } + } + if relevant { + self.scan( + root, + path, + &tokens(call, bytes, deadline)?, + None, + false, + deadline, + &mut included, + &mut env_keys, + unresolved, + )?; + } + } self.env_keys = env_keys.clone(); // An include-bearing macro invoked in this file with an unsupported // matcher or a nested call is already uncertain. More files cannot @@ -426,20 +475,6 @@ impl FileFacts { .push((args.to_vec(), name == "include")); } else { self.direct.push((args.to_vec(), name == "include")); - let empty = BTreeMap::new(); - let target = - expression(args, path, &empty, deadline).and_then(|relative| { - include_path(root, path, &relative, deadline) - }); - match target { - Some(target) => { - included.push(target); - } - None => { - expired(deadline)?; - return Self::refuse(path, false, unresolved); - } - } } } "env" | "option_env" => { @@ -508,7 +543,9 @@ impl SourceInputs { let text = std::str::from_utf8(&bytes).ok()?; if text.contains("include") || text.contains("env") { let mut facts = FileFacts::default(); - facts.parse(root, path, &bytes, deadline, unresolved)?; + let mut names = self.macro_names(); + names.extend(self.searched_names.iter().cloned()); + facts.parse(root, path, &bytes, &names, deadline, unresolved)?; return self.load( root, path, @@ -533,8 +570,16 @@ impl SourceInputs { let mut included = Vec::new(); for (input, rust) in &facts.0.direct { expired(deadline)?; - let target = expression(input, path, &BTreeMap::new(), deadline) - .and_then(|relative| include_path(root, path, &relative, deadline)); + let target = expression(input, path, &BTreeMap::new(), deadline).and_then(|relative| { + include_path( + root, + path, + &relative, + deadline, + &mut self.checked_components, + &mut self.resolved_paths, + ) + }); let Some(target) = target else { expired(deadline)?; return FileFacts::refuse(path, false, unresolved); @@ -563,7 +608,10 @@ impl SourceInputs { expired(deadline)?; if (path.extension().is_some_and(|ext| ext == "rs") || self.rust_sources.contains(path)) - && !self.facts.contains_key(path) + && self + .facts + .get(path) + .is_none_or(|facts| !added.is_subset(&facts.0.scanned_names)) { self.pending.entry(path.clone()).or_insert(None); } @@ -600,7 +648,7 @@ impl SourceInputs { return None; } let mut facts = FileFacts::default(); - facts.parse(root, &path, &bytes, deadline, unresolved)?; + facts.parse(root, &path, &bytes, &names, deadline, unresolved)?; self.load( root, &path, @@ -663,7 +711,16 @@ impl SourceInputs { for (input, rust_source) in &definition.includes { expired(deadline)?; let target = expression(input, &invocation.path, &bindings, deadline).and_then( - |relative| include_path(root, &invocation.path, &relative, deadline), + |relative| { + include_path( + root, + &invocation.path, + &relative, + deadline, + &mut self.checked_components, + &mut self.resolved_paths, + ) + }, ); match target { Some(target) => { @@ -799,6 +856,26 @@ fn string_literal(text: &str, deadline: Instant) -> Option { mod tests { use super::*; + #[test] + fn unrelated_macro_arguments_are_not_collected_as_input_facts() { + let mut facts = FileFacts::default(); + let bytes = br#"// include input prefilter + fn example() { assert_eq!(b"a", b"b"); opaque!(1, 2); } + "#; + facts + .parse( + Path::new("workspace"), + Path::new("workspace/lib.rs"), + bytes, + &BTreeSet::new(), + Instant::now() + std::time::Duration::from_secs(2), + &mut None, + ) + .unwrap(); + assert!(facts.invocations.is_empty()); + assert!(facts.references.is_empty()); + } + #[test] fn unsupported_invoked_matcher_refuses_during_file_scan() { let mut inputs = SourceInputs::default(); From 938c34829c129bbc58f69917d95c179d1d1fa50d Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:55:15 +0100 Subject: [PATCH 020/163] mason: reject shared temp roots and disclose unreadable index folders (#1428) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 06f24c282b61b1280f10da4525135b60d71d02d7) (cherry picked from commit 5df97ab8197f29c1fc3e2e39cf657066ce2dae90) --- .alfonso/release-notes/next/issue-1428.md | 1 + crates/aft-store/src/checkout.rs | 72 +++++- crates/aft-store/src/scan.rs | 118 ++++++++- crates/aft/src/commands/glob.rs | 6 +- crates/aft/src/commands/grep.rs | 11 +- .../aft/src/commands/semantic_search/mod.rs | 20 +- crates/aft/src/data_layer/checkout.rs | 5 +- crates/aft/src/data_layer/surfaces.rs | 227 ++++++++++++++++++ crates/aft/src/list_surfaces.rs | 6 + crates/aft/src/path_identity.rs | 113 ++++++++- crates/aft/src/subc/health.rs | 41 ++++ 11 files changed, 597 insertions(+), 23 deletions(-) create mode 100644 .alfonso/release-notes/next/issue-1428.md diff --git a/.alfonso/release-notes/next/issue-1428.md b/.alfonso/release-notes/next/issue-1428.md new file mode 100644 index 0000000000..77059db838 --- /dev/null +++ b/.alfonso/release-notes/next/issue-1428.md @@ -0,0 +1 @@ +Shared temporary folders no longer start project watchers, indexing, or language-server discovery, regardless of how they are mounted. Projects in their subfolders still work normally. Index builds now skip unreadable folders, report the coverage gap, and retry them on the next rescan instead of failing the whole build. diff --git a/crates/aft-store/src/checkout.rs b/crates/aft-store/src/checkout.rs index faa605a30a..1a4ba3810f 100644 --- a/crates/aft-store/src/checkout.rs +++ b/crates/aft-store/src/checkout.rs @@ -112,6 +112,9 @@ pub struct PathRecord { /// Which content each path of a checkout held at one moment. #[derive(Clone, PartialEq, Eq, Debug)] pub struct CheckoutState { + /// Relative folders the walk could not read, with their OS error text. + /// A rescan retries them rather than treating their absence as complete. + skipped_directories: Arc, String>>, /// The Git tree and index the state was built from, when the checkout /// is in Git. Opaque to the store; compared byte for byte. pub git: Vec, @@ -170,6 +173,7 @@ impl CheckoutState { return Err(StateError::TooLong); } Ok(Self { + skipped_directories: Arc::default(), git, conversion, root: Vec::new(), @@ -189,10 +193,27 @@ impl CheckoutState { self } - /// This state recording the root folder `from` records. + /// This state recording the root folder and walk gaps `from` records. #[must_use] pub fn with_root_of(mut self, from: &Self) -> Self { self.root.clone_from(&from.root); + self.skipped_directories = Arc::clone(&from.skipped_directories); + self + } + + /// Folders the scan could not read, in relative path order. + #[must_use] + pub fn skipped_directories(&self) -> &std::collections::BTreeMap, String> { + &self.skipped_directories + } + + /// Replaces the walk's coverage gaps after a full or partial scan. + #[must_use] + pub fn with_skipped_directories( + mut self, + skipped: std::collections::BTreeMap, String>, + ) -> Self { + self.skipped_directories = Arc::new(skipped); self } @@ -400,6 +421,13 @@ impl CheckoutState { None => out.byte(0), } } + if format >= 4 { + out.usize(self.skipped_directories.len()); + for (path, reason) in self.skipped_directories.iter() { + out.blob(path); + out.str(reason); + } + } out.finish() } @@ -425,8 +453,8 @@ impl CheckoutState { supported: u16::from(STATE_FORMAT), }); } - // Format 2 is format 3 without the root folder. - if format != STATE_FORMAT && format != 2 { + // Format 2 has no root folder; format 3 has no walk gaps. + if !matches!(format, 2..=STATE_FORMAT) { return Err(RecordError::Invalid("unknown checkout state format")); } let git = input.blob().map_err(invalid)?.to_vec(); @@ -440,7 +468,7 @@ impl CheckoutState { .try_into() .map_err(|_| RecordError::Invalid("conversion digest length"))?, ); - let root = if format == STATE_FORMAT { + let root = if format >= 3 { let root = input.blob().map_err(invalid)?.to_vec(); if std::str::from_utf8(&root).is_err() { return Err(RecordError::Invalid("root folder is not UTF-8")); @@ -542,8 +570,18 @@ impl CheckoutState { } _ => return Err(RecordError::Invalid("fragment section flag")), } + let mut skipped_directories = std::collections::BTreeMap::new(); + if format >= 4 { + let count = input.count_of(2).map_err(invalid)?; + for _ in 0..count { + let path = input.blob().map_err(invalid)?.to_vec(); + let reason = input.str().map_err(invalid)?.to_string(); + skipped_directories.insert(path, reason); + } + } input.finish().map_err(invalid)?; Ok(Self { + skipped_directories: Arc::new(skipped_directories), git, conversion, root, @@ -562,7 +600,7 @@ impl CheckoutState { /// flag 2: old readers deliberately treat that flag as unusable and rebuild, /// rather than refusing the whole family on rollback. The record envelope /// and the rest of the state layout are unchanged. -const STATE_FORMAT: u8 = 3; +const STATE_FORMAT: u8 = 4; /// How much of a state record [`newer_state_format`] reads to find the /// payload's format byte: the record header and the payload's string table, @@ -647,6 +685,7 @@ crate::heap_bytes_of_fields!(PathRecord { language, }); crate::heap_bytes_of_fields!(CheckoutState { + skipped_directories, git, conversion, root, @@ -732,6 +771,29 @@ mod tests { ); } + #[test] + fn skipped_folders_survive_encoding_and_rebuilding_and_legacy_states_decode() { + let state = CheckoutState::new(Vec::new(), Digest128::of(b""), vec![record("a", "a")]) + .unwrap() + .with_root(Path::new("/work")) + .with_skipped_directories( + [( + b"locked".to_vec(), + "permission denied (os error 13)".to_string(), + )] + .into(), + ); + let decoded = CheckoutState::decode(&state.encode()).unwrap(); + assert_eq!(decoded, state); + let rebuilt = CheckoutState::new(Vec::new(), state.conversion, Vec::new()) + .unwrap() + .with_root_of(&state); + assert_eq!(rebuilt.skipped_directories(), state.skipped_directories()); + let legacy = CheckoutState::decode(&state.encode_as(3)).unwrap(); + assert_eq!(legacy.root(), state.root()); + assert!(legacy.skipped_directories().is_empty()); + } + #[test] fn round_trips_sorted_and_refuses_duplicates() { let state = CheckoutState::new( diff --git a/crates/aft-store/src/scan.rs b/crates/aft-store/src/scan.rs index 0b8ef9ab66..ef84e0139a 100644 --- a/crates/aft-store/src/scan.rs +++ b/crates/aft-store/src/scan.rs @@ -181,9 +181,11 @@ pub fn scan( previous: Option<&CheckoutState>, sink: &dyn ContentSink, ) -> Result { - let (records, mut counters) = walk_records(root, policy, previous, None, sink)?; + let (records, skipped, mut counters) = walk_records(root, policy, previous, None, sink)?; counters.files = records.len(); - let state = CheckoutState::new(Vec::new(), Digest128::of(b""), records)?.with_root(root); + let state = CheckoutState::new(Vec::new(), Digest128::of(b""), records)? + .with_root(root) + .with_skipped_directories(skipped); Ok(Scan { state, counters }) } @@ -217,13 +219,18 @@ pub fn scan_paths( sink: &dyn ContentSink, ) -> Result { let restrict = Arc::new(Restrict::new(paths)); - let (walked, mut counters) = walk_records( + let (walked, mut skipped, mut counters) = walk_records( root, policy, Some(previous), Some(Arc::clone(&restrict)), sink, )?; + for (path, reason) in previous.skipped_directories() { + if !restrict.covers(path) { + skipped.insert(path.clone(), reason.clone()); + } + } let mut records = previous .paths() .iter() @@ -255,8 +262,9 @@ pub fn scan_paths( drop(walked_by_path); records.extend(walked); counters.files = records.len(); - let state = - CheckoutState::new(previous.git.clone(), previous.conversion, records)?.with_root(root); + let state = CheckoutState::new(previous.git.clone(), previous.conversion, records)? + .with_root(root) + .with_skipped_directories(skipped); Ok(PathScan { state, changed: changed.into_iter().collect(), @@ -324,13 +332,23 @@ impl Restrict { /// Walks `root` under the scan's rules, limited to `restrict` when given, /// and records every member it reaches. +type WalkRecords = ( + Vec, + std::collections::BTreeMap, String>, + ScanCounters, +); + fn walk_records( root: &Path, policy: &ScanPolicy, previous: Option<&CheckoutState>, restrict: Option>, sink: &dyn ContentSink, -) -> Result<(Vec, ScanCounters), ScanError> { +) -> Result { + std::fs::read_dir(root).map_err(|source| ScanError::Walk { + path: root.to_path_buf(), + source, + })?; let common_dir = git_common_dir(root); let mut builder = WalkBuilder::new(root); builder @@ -382,6 +400,7 @@ fn walk_records( }); let records = Mutex::new(Vec::new()); + let skipped = Mutex::new(std::collections::BTreeMap::new()); let walk_error = Mutex::new(None); let counters = Counters::default(); builder.build_parallel().run(|| { @@ -389,13 +408,22 @@ fn walk_records( let entry = match entry { Ok(entry) => entry, Err(error) => { - // A directory that cannot be listed is a hole in the - // state, so the whole scan fails rather than silently - // dropping its files. + // Permission-denied folders are explicit coverage gaps. + // Other walk errors still fail the scan. if let Some(io) = error.io_error() && io.kind() != io::ErrorKind::NotFound { let path = error_path(&error).unwrap_or_else(|| root.to_path_buf()); + if io.kind() == io::ErrorKind::PermissionDenied + && path != root + && let Ok(relative) = relative_bytes(root, &path) + { + skipped + .lock() + .expect("no panics while holding it") + .insert(relative, io.to_string().to_lowercase()); + return WalkState::Continue; + } let mut slot = walk_error.lock().expect("no panics while holding it"); slot.get_or_insert(ScanError::Walk { path, @@ -440,7 +468,11 @@ fn walk_records( return Err(error); } let records = records.into_inner().expect("no panics while holding it"); - Ok((records, counters.finish())) + Ok(( + records, + skipped.into_inner().expect("no panics while holding it"), + counters.finish(), + )) } #[derive(Default)] @@ -840,6 +872,72 @@ mod tests { /// watcher would report, and the paths whose record must change. type Step<'a> = (Box, Vec<&'a str>, Vec<&'a str>); + #[cfg(target_os = "linux")] + #[test] + fn walk_skips_unreadable_folders_and_retries_them() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let locked = root.join("locked"); + std::fs::create_dir(&locked).unwrap(); + std::fs::write(locked.join("hidden.txt"), "needle hidden\n").unwrap(); + std::fs::write(root.join("open.txt"), "needle open\n").unwrap(); + struct Restore(PathBuf); + impl Drop for Restore { + fn drop(&mut self) { + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o700)); + } + } + let restore = Restore(locked.clone()); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0)).unwrap(); + assert_eq!( + std::fs::read_dir(&locked) + .err() + .expect("run without DAC override privileges") + .kind(), + io::ErrorKind::PermissionDenied + ); + let sink = Collect::default(); + let policy = ScanPolicy::default(); + let first = scan(root, &policy, None, &sink).unwrap().state; + assert_eq!(paths(&first), vec!["open.txt"]); + assert_eq!(first.skipped_directories().len(), 1); + assert_eq!( + first.skipped_directories()[b"locked".as_slice()], + "permission denied (os error 13)" + ); + let partial = scan_paths(root, &policy, &first, &[b"open.txt".to_vec()], &sink) + .unwrap() + .state; + assert_eq!(partial.skipped_directories(), first.skipped_directories()); + let retried = scan_paths(root, &policy, &partial, &[b"locked".to_vec()], &sink) + .unwrap() + .state; + assert_eq!(retried.skipped_directories(), first.skipped_directories()); + drop(restore); + let recovered = scan_paths(root, &policy, &retried, &[b"locked".to_vec()], &sink) + .unwrap() + .state; + assert!(recovered.skipped_directories().is_empty()); + assert_eq!(paths(&recovered), vec!["locked/hidden.txt", "open.txt"]); + assert!( + scan(root, &policy, Some(&first), &sink) + .unwrap() + .state + .skipped_directories() + .is_empty() + ); + } + + #[test] + fn walk_a_vanished_root_still_fails() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("gone"); + assert!( + matches!(scan(&missing, &ScanPolicy::default(), None, &Collect::default()), Err(ScanError::Walk { source, .. }) if source.kind() == io::ErrorKind::NotFound) + ); + } + /// The members of `state` with what identifies each, for comparing a /// path scan with a full scan. fn members(state: &CheckoutState) -> Vec { diff --git a/crates/aft/src/commands/glob.rs b/crates/aft/src/commands/glob.rs index 130a41d6d8..1bd03389d6 100644 --- a/crates/aft/src/commands/glob.rs +++ b/crates/aft/src/commands/glob.rs @@ -439,7 +439,11 @@ fn handle_glob_inner(req: &RawRequest, ctx: &AppContext) -> Response { } } - Response::success(&req.id, body) + let mut response = Response::success(&req.id, body); + if source == "index" { + crate::data_layer::surfaces::disclose_index_gaps(ctx, &search_roots, &mut response); + } + response } fn named_ignored_root( diff --git a/crates/aft/src/commands/grep.rs b/crates/aft/src/commands/grep.rs index 5572290677..db1b333040 100644 --- a/crates/aft/src/commands/grep.rs +++ b/crates/aft/src/commands/grep.rs @@ -516,7 +516,16 @@ fn handle_grep_inner(req: &RawRequest, ctx: &AppContext) -> Response { } } - Response::success(&req.id, body) + let mut response = Response::success(&req.id, body); + if matches!(result.index_status, crate::search_index::IndexStatus::Ready) { + let roots = scope + .roots + .iter() + .map(|root| root.search_root.clone()) + .collect::>(); + crate::data_layer::surfaces::disclose_index_gaps(ctx, &roots, &mut response); + } + response } /// `{"trigram": {status, reason}}` for grep/glob responses, from the shared diff --git a/crates/aft/src/commands/semantic_search/mod.rs b/crates/aft/src/commands/semantic_search/mod.rs index 8e53456f3b..b919d3d859 100644 --- a/crates/aft/src/commands/semantic_search/mod.rs +++ b/crates/aft/src/commands/semantic_search/mod.rs @@ -660,6 +660,7 @@ pub fn handle_semantic_search(req: &RawRequest, ctx: &AppContext) -> Response { response.data["text"] = serde_json::json!(format!("{line}\n\n{output}")); } } + disclose_skipped_folders(req, ctx, &mut response); response } @@ -667,11 +668,26 @@ pub fn handle_semantic_search(req: &RawRequest, ctx: &AppContext) -> Response { /// must still run the bounded exact/lexical scan while the indexes warm. pub(crate) fn handle_parent_child_search(req: &RawRequest, ctx: &AppContext) -> Response { let config = ctx.config(); - handle_semantic_search_with_disk_fallback( + let mut response = handle_semantic_search_with_disk_fallback( req, ctx, config.indexes.trigram || config.indexes.semantic, - ) + ); + disclose_skipped_folders(req, ctx, &mut response); + response +} + +fn disclose_skipped_folders(req: &RawRequest, ctx: &AppContext, response: &mut Response) { + let Some(root) = ctx.config().project_root.clone() else { + return; + }; + let scope = req + .params + .get("path") + .and_then(Value::as_str) + .map_or_else(|| root.clone(), |path| root.join(path)); + let scope = std::fs::canonicalize(&scope).unwrap_or(scope); + crate::data_layer::surfaces::disclose_index_gaps(ctx, &[scope], response); } thread_local! { diff --git a/crates/aft/src/data_layer/checkout.rs b/crates/aft/src/data_layer/checkout.rs index c066ee2ac8..17899a5dae 100644 --- a/crates/aft/src/data_layer/checkout.rs +++ b/crates/aft/src/data_layer/checkout.rs @@ -1235,9 +1235,12 @@ impl Checkout { ) .map_err(|error| DataError::Scan(error.to_string()))?; let changed = changed_members(¤t, &scanned.state); + let mut retry = changed.iter().cloned().collect::>(); + retry.extend(current.skipped_directories().keys().cloned()); + retry.extend(scanned.state.skipped_directories().keys().cloned()); // Recording scans each path again under the same rules, so a change // recorded meanwhile is taken from the disk once more, not lost. - self.record_paths(&changed)?; + self.record_paths(&retry.into_iter().collect::>())?; Ok(changed.len()) } diff --git a/crates/aft/src/data_layer/surfaces.rs b/crates/aft/src/data_layer/surfaces.rs index 4a14ec2791..94c6211a87 100644 --- a/crates/aft/src/data_layer/surfaces.rs +++ b/crates/aft/src/data_layer/surfaces.rs @@ -150,6 +150,8 @@ impl PlaneReport { /// surface. #[derive(Clone, Debug)] pub(crate) struct DataStoreReport { + pub(crate) skipped_folders: usize, + pub(crate) skipped_folder_note: Option, pub(crate) text: PlaneReport, pub(crate) callgraph: PlaneReport, /// Paths of the checkout as last published; `None` before anything is. @@ -182,6 +184,8 @@ pub(crate) struct DataStoreReport { impl DataStoreReport { fn both(plane: PlaneReport) -> Self { Self { + skipped_folders: 0, + skipped_folder_note: None, text: plane.clone(), callgraph: plane, files: None, @@ -221,6 +225,8 @@ impl DataStoreReport { .into_iter() .find(|plane| plane.reason.is_some()); DataStoreHealth { + skipped_folders: self.skipped_folders, + skipped_folder_note: self.skipped_folder_note.clone(), reason: plane_reason.and_then(|plane| plane.reason), error: plane_reason.and_then(|plane| plane.error.clone()), pending_edits: self.pending_edits, @@ -238,6 +244,9 @@ impl DataStoreReport { /// The `data_store` object of status and of each root's health. #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct DataStoreHealth { + pub skipped_folders: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub skipped_folder_note: Option, /// Why a plane is off or unavailable; see [`reason`]. #[serde(skip_serializing_if = "Option::is_none")] pub reason: Option<&'static str>, @@ -337,6 +346,107 @@ pub fn text_plane_ready(ctx: &AppContext) -> Option { } } +const INDEX_GAP_NOTE: &str = + "(Index skipped {folders}; results may miss files in them:\n{paths}\n)"; +const INDEX_GAP_PREVIEW: usize = 3; + +fn folder_count(count: usize) -> String { + format!( + "{count} unreadable folder{}", + if count == 1 { "" } else { "s" } + ) +} + +pub(crate) fn skipped_folder_health_note(root: &Path, count: usize) -> String { + format!("{}: index skipped {}", root.display(), folder_count(count)) +} + +/// Report only gaps intersecting the request's scope, under its read policy. +/// The preview is bounded independently of the number of skipped folders. +pub(crate) fn disclose_index_gaps( + ctx: &AppContext, + roots: &[PathBuf], + response: &mut crate::protocol::Response, +) { + if !response.success { + return; + } + let views = if let Some(session) = super::parent_session::session_for(ctx) { + let children = session.children(); + let children = children + .iter() + .filter(|child| { + roots + .iter() + .any(|scope| child.root.starts_with(scope) || scope.starts_with(&child.root)) + }) + .map(|child| child.root.as_path()) + .collect::>(); + super::parent::text_planes(ctx, session.root(), &children) + .into_iter() + .filter_map(|plane| match plane { + super::parent::Plane::Ready(view) => Some((*view).clone()), + _ => None, + }) + .collect::>() + } else if let Some(DataCheckoutProbe::Bound(checkout)) = ctx.probe_data_checkout() { + checkout.view().into_iter().collect::>() + } else { + return; + }; + let fence = crate::context::confinement::capture(); + let mut count = 0; + let mut gaps = Vec::new(); + let mut lines = Vec::new(); + for view in &views { + for (path, reason) in view.current_state().skipped_directories() { + let path = view.root().join(String::from_utf8_lossy(path).as_ref()); + if !roots + .iter() + .any(|scope| path.starts_with(scope) || scope.starts_with(&path)) + || !fence.admits_walk_entry(&path) + { + continue; + } + count += 1; + if gaps.len() < INDEX_GAP_PREVIEW { + lines.push(format!("{}: {reason}", path.display())); + gaps.push( + serde_json::json!({"kind": "unreadable_folder", "path": path, "reason": reason}), + ); + } + } + } + if count == 0 { + return; + } + let omitted = count - gaps.len(); + if omitted > 0 { + lines.push(format!("and {omitted} more")); + } + let note = INDEX_GAP_NOTE + .replace("{folders}", &folder_count(count)) + .replace("{paths}", &lines.join("\n")); + let body = &mut response.data; + body["complete"] = Value::Bool(false); + body["skipped_folders"] = serde_json::json!(count); + body["skipped_folders_omitted"] = serde_json::json!(omitted); + let existing = body + .as_object_mut() + .expect("response body") + .entry("gaps") + .or_insert_with(|| Value::Array(Vec::new())); + if let Some(existing) = existing.as_array_mut() { + existing.extend(gaps); + } + let text = body["text"].as_str().unwrap_or_default(); + body["text"] = Value::String(if text.is_empty() { + note + } else { + format!("{note}\n\n{text}") + }); +} + pub(crate) fn report(ctx: &AppContext) -> Option { Some(match ctx.probe_data_checkout()? { DataCheckoutProbe::NotBound => { @@ -408,6 +518,15 @@ pub(crate) fn report_of(checkout: &DataCheckout) -> DataStoreReport { Some(checkout.checkout().family_dir().to_path_buf()), ); report.meaning = meaning_report(checkout); + report.skipped_folders = checkout + .view() + .map_or(0, |view| view.current_state().skipped_directories().len()); + if report.skipped_folders > 0 { + report.skipped_folder_note = Some(skipped_folder_health_note( + checkout.checkout().root(), + report.skipped_folders, + )); + } report } @@ -714,6 +833,8 @@ fn report_from( PlaneReport::of(READY) }; DataStoreReport { + skipped_folders: 0, + skipped_folder_note: None, text, callgraph, files, @@ -1347,6 +1468,112 @@ mod tests { context_with(root, storage, |_| {}) } + #[cfg(target_os = "linux")] + #[test] + fn unreadable_folders_publish_with_health_and_query_gaps_then_rescan_recovers() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap(); + let ctx = context_with(&root, storage.path(), |config| { + config.indexes.semantic = false + }); + write(&root, "open.txt", "needle open\n"); + write(&root, "locked/hidden.txt", "needle hidden\n"); + let locked = root.join("locked"); + struct Restore(PathBuf); + impl Drop for Restore { + fn drop(&mut self) { + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o700)); + } + } + let restore = Restore(locked.clone()); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0)).unwrap(); + assert_eq!( + std::fs::read_dir(&locked) + .err() + .expect("run without DAC override privileges") + .kind(), + std::io::ErrorKind::PermissionDenied + ); + serve_text_index_for_test(&ctx); + let health = report(&ctx).unwrap().health(); + assert_eq!(health.skipped_folders, 1); + assert_eq!( + health.skipped_folder_note, + Some(format!( + "{}: index skipped 1 unreadable folder", + root.display() + )) + ); + assert_eq!(health.reason, None); + assert!(health.failure.is_none()); + let request = |command: &str| { + serde_json::from_value(serde_json::json!({ + "id": format!("gap-{command}"), "command": command, + "pattern": if command == "glob" { "**/*.txt" } else { "needle" }, + "query": "needle", + })) + .unwrap() + }; + let grep = crate::commands::grep::handle_grep(&request("grep"), &ctx); + let glob = crate::commands::glob::handle_glob(&request("glob"), &ctx); + let search = crate::commands::semantic_search::handle_semantic_search( + &request("semantic_search"), + &ctx, + ); + let expected = format!("(Index skipped 1 unreadable folder; results may miss files in them:\n{}: permission denied (os error 13)\n)", locked.display()); + for response in [&grep, &glob, &search] { + assert!(response.success, "{:?}", response.data); + assert_eq!(response.data["complete"], false, "{:?}", response.data); + assert_eq!(response.data["skipped_folders"], 1); + assert_eq!( + response.data["gaps"][0], + serde_json::json!({"kind":"unreadable_folder", "path": locked, "reason":"permission denied (os error 13)"}) + ); + assert_eq!( + response.data["text"] + .as_str() + .unwrap() + .split("\n\n") + .next() + .unwrap(), + expected + ); + assert!(response.data["text"].as_str().unwrap().contains("open.txt")); + } + assert_eq!(grep.data["index_status"], "Ready"); + let mut scoped = crate::protocol::Response::success( + "scope", + serde_json::json!({"text":"unchanged", "complete":true}), + ); + disclose_index_gaps(&ctx, &[root.join("src")], &mut scoped); + assert_eq!(scoped.data["text"], "unchanged"); + drop(restore); + ctx.data_checkout().unwrap().checkout().rescan().unwrap(); + let recovered = crate::commands::grep::handle_grep(&request("grep"), &ctx); + assert!(recovered.success, "{:?}", recovered.data); + assert!( + recovered.data["text"] + .as_str() + .unwrap() + .contains("hidden.txt"), + "{:?}", + recovered.data + ); + assert!(recovered.data["skipped_folders"].is_null()); + assert_eq!(report(&ctx).unwrap().health().skipped_folders, 0); + } + + #[test] + fn unreadable_folder_notes_use_the_approved_plural_wording() { + assert_eq!( + skipped_folder_health_note(Path::new("/work"), 2), + "/work: index skipped 2 unreadable folders" + ); + assert_eq!(INDEX_GAP_NOTE.replace("{folders}", &folder_count(2)).replace("{paths}", "a: permission denied\nb: operation not permitted"), "(Index skipped 2 unreadable folders; results may miss files in them:\na: permission denied\nb: operation not permitted\n)"); + } + /// As [`context`], with `edit` applied to its configuration. fn context_with( root: &Path, diff --git a/crates/aft/src/list_surfaces.rs b/crates/aft/src/list_surfaces.rs index 3e197ae4f3..f447e7fa72 100644 --- a/crates/aft/src/list_surfaces.rs +++ b/crates/aft/src/list_surfaces.rs @@ -1340,6 +1340,12 @@ pub static EXCLUSIONS: &[ExclusionEntry] = &[ location_or_primitive: "SNIPPET_LINE_MAX_CHARS", reason: "incremental implementation of the existing per-line text window, with exactly the same ellipsis and truncation marker; bounds retained characters, not result rows", }, + ExclusionEntry { + file: "data_layer/surfaces.rs", + enclosing_item: "disclose_index_gaps", + location_or_primitive: "INDEX_GAP_PREVIEW", + reason: "coverage-gap preview, not a findings page; skipped_folders counts every relevant unreadable folder and skipped_folders_omitted counts names beyond the preview", + }, ]; /// Look up a surface by command, mode, and list id. diff --git a/crates/aft/src/path_identity.rs b/crates/aft/src/path_identity.rs index 622adbf5bf..2714db5b2a 100644 --- a/crates/aft/src/path_identity.rs +++ b/crates/aft/src/path_identity.rs @@ -112,12 +112,32 @@ fn directory_is_system_tree(directory: &Path) -> bool { /// directory scopes, which also exclude root-level containers. /// Unknown protected-directory identities must not admit background work. pub(crate) fn project_root_is_narrow(directory: &Path, home: &Path, volume: &Path) -> Option { - if !project_root_path_is_narrow(directory, home, volume) { + if !project_root_path_is_narrow(directory, home, volume) + || project_root_is_shared_temp(directory) + { return Some(false); } protected_directory_alias(directory, home, volume).map(|protected| !protected) } +/// Shared temporary roots hold other applications' files on any mount layout. +/// Compare canonical paths, not prefixes, so projects below them stay enabled. +fn project_root_is_shared_temp(directory: &Path) -> bool { + let directory = crate::access_policy::policy_path(directory); + let platform_temp = std::env::temp_dir(); + let roots = std::iter::once(platform_temp.as_path()); + #[cfg(unix)] + let roots = roots.chain([ + Path::new("/tmp"), + Path::new("/var/tmp"), + Path::new("/private/tmp"), + Path::new("/private/var/tmp"), + ]); + roots + .filter_map(|root| ProjectRootId::from_path(root).ok()) + .any(|root| crate::access_policy::policy_path(&root.into_path_buf()) == directory) +} + fn project_root_path_is_narrow(directory: &Path, home: &Path, volume: &Path) -> bool { directory.is_absolute() && directory.parent().is_some() @@ -434,6 +454,74 @@ pub(crate) fn lexical_normalize(path: &Path) -> PathBuf { #[cfg(test)] mod tests { + #[test] + fn platform_temp_root_is_broad_but_tempdir_projects_are_narrow() { + let root = super::ProjectRootId::from_path(&std::env::temp_dir()) + .unwrap() + .into_path_buf(); + assert!(super::project_root_is_shared_temp(&root)); + let project = tempfile::tempdir().unwrap(); + let project = project.path().canonicalize().unwrap(); + assert!(!super::project_root_is_shared_temp(&project)); + assert_eq!( + super::project_root_is_narrow(&root, &project, root.ancestors().last().unwrap()), + Some(false) + ); + } + + #[cfg(unix)] + #[test] + fn named_shared_temp_roots_use_canonical_identities() { + for name in ["/tmp", "/var/tmp"] { + let root = super::ProjectRootId::from_path(std::path::Path::new(name)) + .unwrap() + .into_path_buf(); + assert!(super::project_root_is_shared_temp(&root), "{name}"); + assert!(!super::project_root_is_shared_temp( + &root.join("my-scratch-project") + )); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_private_temp_forms_and_platform_temp_are_broad() { + for name in [ + std::path::PathBuf::from("/private/tmp"), + std::path::PathBuf::from("/private/var/tmp"), + std::env::temp_dir(), + ] { + let root = super::ProjectRootId::from_path(&name) + .unwrap() + .into_path_buf(); + assert!( + super::project_root_is_shared_temp(&root), + "{}", + name.display() + ); + assert_eq!( + super::project_root_is_narrow(&root, &root.join("home"), std::path::Path::new("/")), + Some(false) + ); + assert!(!super::project_root_is_shared_temp( + &root.join("my-scratch-project") + )); + } + } + + #[cfg(windows)] + #[test] + fn windows_temp_root_uses_the_canonical_platform_identity() { + let temp = std::env::temp_dir(); + let root = super::ProjectRootId::from_path(&temp) + .unwrap() + .into_path_buf(); + assert!(super::project_root_is_shared_temp(&root)); + assert!(!super::project_root_is_shared_temp( + &root.join("my-scratch-project") + )); + } + #[test] fn project_root_volume_children_are_not_permission_scopes() { let volume = tempfile::tempdir().unwrap(); @@ -576,7 +664,8 @@ mod tests { #[cfg(target_os = "linux")] mod mount_table { use super::super::{ - mount_root_in, project_root_path_is_narrow, project_root_volume_with, MountRoot, + mount_root_in, project_root_is_shared_temp, project_root_path_is_narrow, + project_root_volume_with, MountRoot, }; use std::path::{Path, PathBuf}; @@ -589,7 +678,25 @@ mod tests { project_root_volume_with(root, root.to_path_buf(), Some(home), read, |_| { Some(PathBuf::from("/")) }); - volume.is_some_and(|volume| project_root_path_is_narrow(root, home, &volume)) + volume.is_some_and(|volume| { + project_root_path_is_narrow(root, home, &volume) + && !project_root_is_shared_temp(root) + }) + } + + #[test] + fn shared_temp_roots_are_broad_on_bind_and_tmpfs_mounts() { + for root in ["/tmp", "/var/tmp"] { + assert!(!narrow( + root, + &format!("1801 1748 8:1 /tmp {root} rw - ext4 /dev/sda1 rw") + )); + assert!(!narrow( + root, + &format!("1801 1748 0:45 / {root} rw - tmpfs tmpfs rw") + )); + assert!(narrow(&format!("{root}/my-scratch-project"), &format!("1801 1748 8:1 /tmp/project {root}/my-scratch-project rw - ext4 /dev/sda1 rw"))); + } } fn narrow(root: &str, mount: &str) -> bool { diff --git a/crates/aft/src/subc/health.rs b/crates/aft/src/subc/health.rs index 4785802fb9..66870417e0 100644 --- a/crates/aft/src/subc/health.rs +++ b/crates/aft/src/subc/health.rs @@ -2003,6 +2003,7 @@ fn ordinary_ready_health_root(root: &Value) -> bool { ) }) && root["data_store"]["failure"].is_null() + && root["data_store"]["skipped_folders"].as_u64().unwrap_or(0) == 0 && root["suspended_domains"] .as_array() .is_none_or(Vec::is_empty) @@ -2096,6 +2097,19 @@ fn abbreviate_health_root(root: &mut Value) -> bool { *path = abbreviated_health_path(path); } } + if let (Some(root_path), Some(count)) = ( + row.get("project_root").and_then(Value::as_str), + row.get("data_store") + .and_then(|store| store["skipped_folders"].as_u64()), + ) { + if count > 0 { + let note = crate::data_layer::surfaces::skipped_folder_health_note( + Path::new(root_path), + count as usize, + ); + row["data_store"]["skipped_folder_note"] = Value::String(note); + } + } if let Some(Value::String(refusal)) = row.get_mut("standing_refusal") { *refusal = abbreviated_health_refusal(refusal); } @@ -5010,6 +5024,8 @@ mod tests { failure: Option<(&'static str, bool)>, ) -> crate::data_layer::surfaces::DataStoreHealth { crate::data_layer::surfaces::DataStoreHealth { + skipped_folders: 0, + skipped_folder_note: None, reason, error: None, pending_edits: 0, @@ -5025,6 +5041,31 @@ mod tests { } } + #[test] + fn data_store_unreadable_folders_are_informational_and_keep_budgeted_identity() { + let mut store = data_store_health(None, None); + store.skipped_folders = 2; + let root_path = format!("/work/{}/project", "long/".repeat(100)); + store.skipped_folder_note = Some(crate::data_layer::surfaces::skipped_folder_health_note( + Path::new(&root_path), + 2, + )); + assert_eq!(data_store_failure_reason(&store), None); + let failed = DataStoreFailedRoots::collect([(root_path.as_str(), &store)]); + let state = root_health_rollup_state(0, 0, 1, None); + assert_eq!(with_data_store_failures(state.clone(), &failed), state); + let mut root = json!({"project_root": root_path, "data_store": store}); + assert!(abbreviate_health_root(&mut root)); + assert_eq!(root["data_store"]["skipped_folders"], 2); + assert_eq!( + root["data_store"]["skipped_folder_note"], + format!( + "{}: index skipped 2 unreadable folders", + abbreviated_health_path(&root_path) + ) + ); + } + #[test] fn failed_index_store_roots_are_named_in_path_order_with_reasons_up_to_five() { use crate::data_layer::surfaces::reason; From 31935142ced6a4c4acfefd9ac3b49dc7b1653a63 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:01:33 +0100 Subject: [PATCH 021/163] mason: repair gap compile checks and preserve vanished-root errors (#1428) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 74239d6e9d88d5e00fab914ca2dffc6e7e7af2b8) (cherry picked from commit 1a5429a24804f4352d8268ba53f5bab81a4b4248) --- crates/aft-store/src/checkout.rs | 1 + crates/aft-store/src/scan.rs | 11 ++++++----- crates/aft/src/commands/semantic_search/mod.rs | 2 +- crates/aft/src/data_layer/surfaces.rs | 7 ++++++- crates/aft/src/path_identity.rs | 5 +++-- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/crates/aft-store/src/checkout.rs b/crates/aft-store/src/checkout.rs index 1a4ba3810f..ca11fa9622 100644 --- a/crates/aft-store/src/checkout.rs +++ b/crates/aft-store/src/checkout.rs @@ -600,6 +600,7 @@ impl CheckoutState { /// flag 2: old readers deliberately treat that flag as unusable and rebuild, /// rather than refusing the whole family on rollback. The record envelope /// and the rest of the state layout are unchanged. +/// Format 4 appends the skipped-directory paths and their permission errors. const STATE_FORMAT: u8 = 4; /// How much of a state record [`newer_state_format`] reads to find the diff --git a/crates/aft-store/src/scan.rs b/crates/aft-store/src/scan.rs index ef84e0139a..ee133abfaf 100644 --- a/crates/aft-store/src/scan.rs +++ b/crates/aft-store/src/scan.rs @@ -330,14 +330,14 @@ impl Restrict { } } -/// Walks `root` under the scan's rules, limited to `restrict` when given, -/// and records every member it reaches. type WalkRecords = ( Vec, std::collections::BTreeMap, String>, ScanCounters, ); +/// Walks `root` under the scan's rules, limited to `restrict` when given, +/// and records every member it reaches. fn walk_records( root: &Path, policy: &ScanPolicy, @@ -410,10 +410,11 @@ fn walk_records( Err(error) => { // Permission-denied folders are explicit coverage gaps. // Other walk errors still fail the scan. - if let Some(io) = error.io_error() - && io.kind() != io::ErrorKind::NotFound - { + if let Some(io) = error.io_error() { let path = error_path(&error).unwrap_or_else(|| root.to_path_buf()); + if io.kind() == io::ErrorKind::NotFound && path != root { + return WalkState::Continue; + } if io.kind() == io::ErrorKind::PermissionDenied && path != root && let Ok(relative) = relative_bytes(root, &path) diff --git a/crates/aft/src/commands/semantic_search/mod.rs b/crates/aft/src/commands/semantic_search/mod.rs index b919d3d859..96ed2f63f0 100644 --- a/crates/aft/src/commands/semantic_search/mod.rs +++ b/crates/aft/src/commands/semantic_search/mod.rs @@ -684,7 +684,7 @@ fn disclose_skipped_folders(req: &RawRequest, ctx: &AppContext, response: &mut R let scope = req .params .get("path") - .and_then(Value::as_str) + .and_then(serde_json::Value::as_str) .map_or_else(|| root.clone(), |path| root.join(path)); let scope = std::fs::canonicalize(&scope).unwrap_or(scope); crate::data_layer::surfaces::disclose_index_gaps(ctx, &[scope], response); diff --git a/crates/aft/src/data_layer/surfaces.rs b/crates/aft/src/data_layer/surfaces.rs index 94c6211a87..f15893871f 100644 --- a/crates/aft/src/data_layer/surfaces.rs +++ b/crates/aft/src/data_layer/surfaces.rs @@ -244,6 +244,7 @@ impl DataStoreReport { /// The `data_store` object of status and of each root's health. #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct DataStoreHealth { + #[serde(skip_serializing_if = "no_skipped_folders")] pub skipped_folders: usize, #[serde(skip_serializing_if = "Option::is_none")] pub skipped_folder_note: Option, @@ -258,6 +259,10 @@ pub struct DataStoreHealth { pub failure: Option, } +fn no_skipped_folders(count: &usize) -> bool { + *count == 0 +} + /// The last failed build or refresh of a checkout. #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct DataStoreFailure { @@ -385,7 +390,7 @@ pub(crate) fn disclose_index_gaps( super::parent::text_planes(ctx, session.root(), &children) .into_iter() .filter_map(|plane| match plane { - super::parent::Plane::Ready(view) => Some((*view).clone()), + super::parent_session::Plane::Ready(view) => Some((*view).clone()), _ => None, }) .collect::>() diff --git a/crates/aft/src/path_identity.rs b/crates/aft/src/path_identity.rs index 2714db5b2a..a2b5107e5a 100644 --- a/crates/aft/src/path_identity.rs +++ b/crates/aft/src/path_identity.rs @@ -133,9 +133,10 @@ fn project_root_is_shared_temp(directory: &Path) -> bool { Path::new("/private/tmp"), Path::new("/private/var/tmp"), ]); - roots + let shared = roots .filter_map(|root| ProjectRootId::from_path(root).ok()) - .any(|root| crate::access_policy::policy_path(&root.into_path_buf()) == directory) + .any(|root| crate::access_policy::policy_path(&root.into_path_buf()) == directory); + shared } fn project_root_path_is_narrow(directory: &Path, home: &Path, volume: &Path) -> bool { From 763ff2ad62945d000d949afeb8da7f6a55cd0282 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:32:46 +0100 Subject: [PATCH 022/163] mason: treat unreadable checkout roots as retryable index gaps (#1428) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit b3358efd10fac4c8ab5bb7b0f019a775d6f14dbc) (cherry picked from commit 291b6c6dce10dcd2389ca18e12375971fadcca17) --- crates/aft-store/src/scan.rs | 60 +++++++++++++++++++++--- crates/aft/src/data_layer/surfaces.rs | 67 +++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 7 deletions(-) diff --git a/crates/aft-store/src/scan.rs b/crates/aft-store/src/scan.rs index ee133abfaf..484a86df95 100644 --- a/crates/aft-store/src/scan.rs +++ b/crates/aft-store/src/scan.rs @@ -203,6 +203,7 @@ pub struct PathScan { /// Brings `previous` up to date for `paths` (relative, such as a watcher or /// an edit reports them): each path, and everything under it when it is a /// folder, is walked again under exactly the rules a full [`scan`] applies. +/// An empty relative path names the root and rescans the whole checkout. /// A path that is gone, ignored, or inside a skipped folder leaves the /// state; a folder that appeared is walked whole. Every other record is kept /// as it is. Files are read only when their stat no longer proves the @@ -299,14 +300,13 @@ impl Restrict { } path }) - .filter(|path| !path.is_empty()) .collect(), } } /// Whether `path` is one of the paths or under one of them. fn covers(&self, path: &[u8]) -> bool { - if self.paths.contains(path) { + if self.paths.contains(b"".as_slice()) || self.paths.contains(path) { return true; } path.iter() @@ -345,10 +345,19 @@ fn walk_records( restrict: Option>, sink: &dyn ContentSink, ) -> Result { - std::fs::read_dir(root).map_err(|source| ScanError::Walk { - path: root.to_path_buf(), - source, - })?; + if let Err(source) = std::fs::read_dir(root) { + if source.kind() == io::ErrorKind::PermissionDenied { + return Ok(( + Vec::new(), + [(Vec::new(), source.to_string().to_lowercase())].into(), + ScanCounters::default(), + )); + } + return Err(ScanError::Walk { + path: root.to_path_buf(), + source, + }); + } let common_dir = git_common_dir(root); let mut builder = WalkBuilder::new(root); builder @@ -416,7 +425,6 @@ fn walk_records( return WalkState::Continue; } if io.kind() == io::ErrorKind::PermissionDenied - && path != root && let Ok(relative) = relative_bytes(root, &path) { skipped @@ -939,6 +947,44 @@ mod tests { ); } + #[cfg(target_os = "linux")] + #[test] + fn walk_an_unreadable_root_is_a_gap_and_is_retried_after_access_returns() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + std::fs::write(root.join("readable-later.txt"), "needle\n").unwrap(); + struct Restore(PathBuf); + impl Drop for Restore { + fn drop(&mut self) { + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o700)); + } + } + let restore = Restore(root.to_path_buf()); + std::fs::set_permissions(root, std::fs::Permissions::from_mode(0)).unwrap(); + assert_eq!( + std::fs::read_dir(root) + .err() + .expect("run without DAC override privileges") + .kind(), + io::ErrorKind::PermissionDenied + ); + let sink = Collect::default(); + let policy = ScanPolicy::default(); + let first = scan(root, &policy, None, &sink).unwrap().state; + assert!(first.paths().is_empty()); + assert_eq!( + first.skipped_directories()[b"".as_slice()], + "permission denied (os error 13)" + ); + drop(restore); + let recovered = scan_paths(root, &policy, &first, &[Vec::new()], &sink) + .unwrap() + .state; + assert!(recovered.skipped_directories().is_empty()); + assert_eq!(paths(&recovered), vec!["readable-later.txt"]); + } + /// The members of `state` with what identifies each, for comparing a /// path scan with a full scan. fn members(state: &CheckoutState) -> Vec { diff --git a/crates/aft/src/data_layer/surfaces.rs b/crates/aft/src/data_layer/surfaces.rs index f15893871f..4d6a75b0d6 100644 --- a/crates/aft/src/data_layer/surfaces.rs +++ b/crates/aft/src/data_layer/surfaces.rs @@ -1579,6 +1579,73 @@ mod tests { assert_eq!(INDEX_GAP_NOTE.replace("{folders}", &folder_count(2)).replace("{paths}", "a: permission denied\nb: operation not permitted"), "(Index skipped 2 unreadable folders; results may miss files in them:\na: permission denied\nb: operation not permitted\n)"); } + #[cfg(target_os = "linux")] + #[test] + fn unreadable_root_publishes_a_named_gap_and_rescan_retries_after_access_returns() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap(); + let ctx = context_with(&root, storage.path(), |config| { + config.indexes.semantic = false + }); + write(&root, "readable-later.txt", "needle\n"); + let hold = super::super::runtime::hold_builds_for_test(&root); + bind_for_test(&ctx); + struct Restore(PathBuf); + impl Drop for Restore { + fn drop(&mut self) { + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o700)); + } + } + let restore = Restore(root.clone()); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0)).unwrap(); + assert_eq!( + std::fs::read_dir(&root) + .err() + .expect("run without DAC override privileges") + .kind(), + std::io::ErrorKind::PermissionDenied + ); + drop(hold); + serve_text_index_for_test(&ctx); + let checkout = ctx.data_checkout().unwrap(); + assert!(checkout.view().unwrap().current_state().paths().is_empty()); + let health = report(&ctx).unwrap().health(); + assert_eq!( + health.skipped_folder_note, + Some(format!( + "{}: index skipped 1 unreadable folder", + root.display() + )) + ); + assert_eq!(health.reason, None); + assert!(health.failure.is_none()); + let req = serde_json::from_value( + serde_json::json!({"id":"root-gap", "command":"grep", "pattern":"needle"}), + ) + .unwrap(); + let response = crate::commands::grep::handle_grep(&req, &ctx); + assert!(response.success, "{:?}", response.data); + assert_eq!(response.data["complete"], false); + assert_eq!(response.data["gaps"][0]["path"], serde_json::json!(root)); + assert_eq!(response.data["text"].as_str().unwrap().split("\n\n").next().unwrap(), format!("(Index skipped 1 unreadable folder; results may miss files in them:\n{}: permission denied (os error 13)\n)", root.display())); + drop(restore); + checkout.checkout().rescan().unwrap(); + let recovered = crate::commands::grep::handle_grep(&req, &ctx); + assert!(recovered.success, "{:?}", recovered.data); + assert!( + recovered.data["text"] + .as_str() + .unwrap() + .contains("readable-later.txt"), + "{:?}", + recovered.data + ); + assert!(recovered.data["skipped_folders"].is_null()); + assert_eq!(report(&ctx).unwrap().health().skipped_folders, 0); + } + /// As [`context`], with `edit` applied to its configuration. fn context_with( root: &Path, From 42aff7b0dea0256f679e8dca9286e5336ce81cbf Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:43:47 +0100 Subject: [PATCH 023/163] mason: preserve canonical root spelling in index gaps (#1428) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit aa2bae50840971b50493a20a9e33e1368bb1504c) (cherry picked from commit 2c62e235dbb8b1a8b3e239a3c0f2b494c20f3414) --- crates/aft/src/data_layer/surfaces.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/aft/src/data_layer/surfaces.rs b/crates/aft/src/data_layer/surfaces.rs index 4d6a75b0d6..45ea40e9f1 100644 --- a/crates/aft/src/data_layer/surfaces.rs +++ b/crates/aft/src/data_layer/surfaces.rs @@ -405,7 +405,11 @@ pub(crate) fn disclose_index_gaps( let mut lines = Vec::new(); for view in &views { for (path, reason) in view.current_state().skipped_directories() { - let path = view.root().join(String::from_utf8_lossy(path).as_ref()); + let path = if path.is_empty() { + view.root().to_path_buf() + } else { + view.root().join(String::from_utf8_lossy(path).as_ref()) + }; if !roots .iter() .any(|scope| path.starts_with(scope) || scope.starts_with(&path)) From 7592138b9de5fb770f4563372f092ebcffae44d1 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:08:20 +0100 Subject: [PATCH 024/163] mason: keep index gaps rebuildable on 0.64 rollback (#1428) Keep STATE_FORMAT at 3 and put skipped folders behind fragment-section flag 3, reusing the unknown-section rebuild behavior. Gap-free states keep the exact 0.64 encoding. A gap-bearing state must now be rejected by the old decoder rather than encoded as a gap-free format-3 state; the persistence test's legacy assertion changes accordingly. Add an independent frozen 0.64 writer oracle and sibling/family rollback checks. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 0034bf9360224fbcb5fa552146977e46883b6aa5) (cherry picked from commit 33f1ef50bb115cd701c0413af3ba368699f52181) --- crates/aft-store/src/checkout.rs | 181 ++++++++++++++++++++++++--- crates/aft-store/src/family/tests.rs | 61 +++++++++ 2 files changed, 228 insertions(+), 14 deletions(-) diff --git a/crates/aft-store/src/checkout.rs b/crates/aft-store/src/checkout.rs index ca11fa9622..3ade44a323 100644 --- a/crates/aft-store/src/checkout.rs +++ b/crates/aft-store/src/checkout.rs @@ -402,6 +402,10 @@ impl CheckoutState { out.str(&record.language); previous = Some(record); } + let has_gaps = !self.skipped_directories.is_empty(); + if has_gaps { + out.byte(3); + } // Flag 2 adds a producer stamp before the optional digests. Older // readers reject this flag as unusable and rebuild the checkout. out.byte(if self.graph_extractor.is_some() { @@ -421,7 +425,7 @@ impl CheckoutState { None => out.byte(0), } } - if format >= 4 { + if has_gaps { out.usize(self.skipped_directories.len()); for (path, reason) in self.skipped_directories.iter() { out.blob(path); @@ -444,6 +448,19 @@ impl CheckoutState { } fn decode_layout(payload: &[u8], stamped_graphs: bool) -> Result { + Self::decode_sections(payload, stamped_graphs, true) + } + + #[cfg(test)] + pub(crate) fn decode_as_v064(payload: &[u8]) -> Result { + Self::decode_sections(payload, true, false) + } + + fn decode_sections( + payload: &[u8], + stamped_graphs: bool, + coverage_gaps: bool, + ) -> Result { let invalid = |error: DecodeError| RecordError::Invalid(error.0); let mut input = Decoder::new(payload).map_err(invalid)?; let format = input.byte().map_err(invalid)?; @@ -453,7 +470,7 @@ impl CheckoutState { supported: u16::from(STATE_FORMAT), }); } - // Format 2 has no root folder; format 3 has no walk gaps. + // Format 2 has no root folder. if !matches!(format, 2..=STATE_FORMAT) { return Err(RecordError::Invalid("unknown checkout state format")); } @@ -543,7 +560,12 @@ impl CheckoutState { } let mut fragments = Vec::new(); let mut graph_extractor = None; - match input.byte().map_err(invalid)? { + let mut flag = input.byte().map_err(invalid)?; + let has_gaps = flag == 3 && coverage_gaps; + if has_gaps { + flag = input.byte().map_err(invalid)?; + } + match flag { 0 => {} flag @ (1 | 2) if flag == 1 || stamped_graphs => { if flag == 2 { @@ -571,7 +593,7 @@ impl CheckoutState { _ => return Err(RecordError::Invalid("fragment section flag")), } let mut skipped_directories = std::collections::BTreeMap::new(); - if format >= 4 { + if has_gaps { let count = input.count_of(2).map_err(invalid)?; for _ in 0..count { let path = input.blob().map_err(invalid)?.to_vec(); @@ -596,12 +618,16 @@ impl CheckoutState { /// the payload's string table. A reader refuses a payload naming a higher /// one, since a newer build wrote it. So every later layout must keep the /// string table's encoding and this byte where they are, and any other -/// change to the layout must raise this number, except the stamped fragment -/// flag 2: old readers deliberately treat that flag as unusable and rebuild, -/// rather than refusing the whole family on rollback. The record envelope -/// and the rest of the state layout are unchanged. -/// Format 4 appends the skipped-directory paths and their permission errors. -const STATE_FORMAT: u8 = 4; +/// change to the layout must raise this number, except rebuildable fragment +/// sections. Fragment-section flag 2 adds a producer stamp: older readers +/// treat an unknown section flag as unusable and rebuild the checkout rather +/// than refusing the whole family on rollback. The record envelope and the +/// rest of the state layout are unchanged. +/// Fragment-section flag 3 uses the same mechanism: it wraps the existing +/// fragment section and appends skipped-directory paths and errors. Readers +/// that do not know this section flag rebuild only that checkout instead of +/// refusing the family. Gap-free states keep their original bytes and flags. +const STATE_FORMAT: u8 = 3; /// How much of a state record [`newer_state_format`] reads to find the /// payload's format byte: the record header and the payload's string table, @@ -699,6 +725,131 @@ crate::heap_bytes_of_fields!(CheckoutState { mod tests { use super::*; + // Frozen from v0.64.0's encode_layout, with its format fixed at 3. + // This oracle must not delegate to the current writer: gap-free bytes + // must remain compatible even as new optional sections are introduced. + fn encode_v064(state: &CheckoutState) -> Vec { + let mut out = Encoder::default(); + out.byte(3); + out.blob(&state.git); + out.blob(state.conversion.as_bytes()); + out.blob(&state.root); + out.usize(state.paths.len()); + let mut previous: Option<&PathRecord> = None; + for record in state.paths.iter() { + let shared = previous.map_or(0, |previous| { + previous + .path + .iter() + .zip(&record.path) + .take_while(|(a, b)| a == b) + .count() + }); + out.usize(shared); + out.blob(&record.path[shared..]); + out.u32(record.mode); + out.byte(record.kind.code()); + out.blob(record.content.digest().as_bytes()); + out.blob(&record.link); + let stat = &record.stat; + let before = previous.map(|previous| &previous.stat); + out.u64(stat.size); + out.u64(zigzag(stat.mtime_ns)); + out.u64(zigzag(stat.ctime_ns.wrapping_sub(stat.mtime_ns))); + put_optional( + &mut out, + stat.birth_ns + .map(|birth| zigzag(birth.wrapping_sub(stat.mtime_ns))), + ); + out.u64(stat.file_id); + out.u64(zigzag( + stat.device + .wrapping_sub(before.map_or(0, |before| before.device)) + .cast_signed(), + )); + put_optional(&mut out, stat.usn); + put_optional(&mut out, stat.journal); + out.u64(zigzag( + stat.observed_ns + .wrapping_sub(before.map_or(0, |before| before.observed_ns)), + )); + out.str(&record.language); + previous = Some(record); + } + out.byte(if state.graph_extractor.is_some() { + 2 + } else { + u8::from(!state.fragments.is_empty()) + }); + if let Some(extractor) = &state.graph_extractor { + out.blob(extractor.as_bytes()); + } + for fragment in state.fragments.iter() { + match fragment { + Some(digest) => { + out.byte(1); + out.blob(digest.as_bytes()); + } + None => out.byte(0), + } + } + out.finish() + } + + #[test] + fn gap_free_states_keep_v064_bytes_for_every_fragment_section() { + for paths in [ + Vec::new(), + vec![record("src/a.rs", "a"), record("src/b.rs", "b")], + ] { + let state = CheckoutState::new(b"git".to_vec(), Digest128::of(b"attrs"), paths) + .unwrap() + .with_root(Path::new("/work")); + let fragments = vec![Some(Digest128::of(b"fragment")); state.paths().len()]; + let states = [ + state.clone(), + state.clone().with_fragments(fragments.clone()).unwrap(), + state.with_fragments_for(fragments, "current").unwrap(), + ]; + for state in states { + assert_eq!(state.encode(), encode_v064(&state)); + assert_eq!( + CheckoutState::decode_as_v064(&state.encode()).unwrap(), + state + ); + } + } + } + + #[test] + fn gap_sections_round_trip_with_each_fragment_section_and_v064_rejects_them() { + for paths in [Vec::new(), vec![record("a.rs", "a")]] { + let state = CheckoutState::new(Vec::new(), Digest128::of(b""), paths).unwrap(); + let fragments = vec![Some(Digest128::of(b"fragment")); state.paths().len()]; + let states = [ + state.clone(), + state.clone().with_fragments(fragments.clone()).unwrap(), + state.with_fragments_for(fragments, "current").unwrap(), + ]; + for state in states { + let state = state.with_skipped_directories( + [( + b"locked".to_vec(), + "permission denied (os error 13)".to_string(), + )] + .into(), + ); + let payload = state.encode(); + assert_eq!(Decoder::new(&payload).unwrap().byte().unwrap(), 3); + assert_eq!(CheckoutState::decode(&payload).unwrap(), state); + assert!(matches!( + CheckoutState::decode_as_v064(&payload), + Err(RecordError::Invalid("fragment section flag")) + )); + } + } + } + fn record(path: &str, content: &str) -> PathRecord { PathRecord { path: path.as_bytes().to_vec(), @@ -773,7 +924,7 @@ mod tests { } #[test] - fn skipped_folders_survive_encoding_and_rebuilding_and_legacy_states_decode() { + fn skipped_folders_survive_encoding_and_rebuilding() { let state = CheckoutState::new(Vec::new(), Digest128::of(b""), vec![record("a", "a")]) .unwrap() .with_root(Path::new("/work")) @@ -790,9 +941,11 @@ mod tests { .unwrap() .with_root_of(&state); assert_eq!(rebuilt.skipped_directories(), state.skipped_directories()); - let legacy = CheckoutState::decode(&state.encode_as(3)).unwrap(); - assert_eq!(legacy.root(), state.root()); - assert!(legacy.skipped_directories().is_empty()); + assert_eq!(Decoder::new(&state.encode()).unwrap().byte().unwrap(), 3); + assert!(matches!( + CheckoutState::decode_sections(&state.encode(), true, false), + Err(RecordError::Invalid("fragment section flag")) + )); } #[test] diff --git a/crates/aft-store/src/family/tests.rs b/crates/aft-store/src/family/tests.rs index 3edc78b415..6163d2b414 100644 --- a/crates/aft-store/src/family/tests.rs +++ b/crates/aft-store/src/family/tests.rs @@ -906,3 +906,64 @@ fn a_checkout_state_payload_an_older_build_wrote_stays_unusable() { "a writer still starts" ); } + +#[test] +fn a_checkout_state_with_gaps_is_unusable_to_v064_without_refusing_its_family() { + use crate::checkout::CheckoutState; + let dir = tempfile::tempdir().unwrap(); + let family = open(dir.path()); + let gap_free = CheckoutState::new(Vec::new(), Digest128::of(b""), Vec::new()).unwrap(); + let gapped = gap_free.clone().with_skipped_directories( + [( + b"locked".to_vec(), + "permission denied (os error 13)".to_string(), + )] + .into(), + ); + let mut writer = family.try_write().unwrap().unwrap(); + let main = writer + .write_checkout_state(checkout("main"), &gapped) + .unwrap(); + let sibling = writer + .write_checkout_state(checkout("sibling"), &gap_free) + .unwrap(); + writer + .publish(&Publication { + add_segments: Vec::new(), + remove_segments: Vec::new(), + states: vec![main, sibling], + retire_checkouts: Vec::new(), + }) + .unwrap(); + drop(writer); + let snapshot = family.snapshot().unwrap(); + assert_eq!( + snapshot.checkout_state(checkout("main")).unwrap().unwrap(), + gapped + ); + let payload = snapshot.read_state(checkout("main")).unwrap().unwrap(); + let error = CheckoutState::decode_as_v064(&payload).unwrap_err(); + assert!(matches!( + error, + RecordError::Invalid("fragment section flag") + )); + let path = family.layout.state( + checkout("main"), + snapshot + .manifest() + .checkout(checkout("main")) + .unwrap() + .number, + ); + let error = FamilyError::record(&path, error); + assert!(error.is_unusable_file(), "{error}"); + assert_eq!(error.newer_format(), None); + let sibling = snapshot.read_state(checkout("sibling")).unwrap().unwrap(); + assert_eq!(CheckoutState::decode_as_v064(&sibling).unwrap(), gap_free); + drop(snapshot); + family.check_formats().unwrap(); + assert!( + family.try_write().unwrap().is_some(), + "an unusable checkout must not block its family writer" + ); +} From 81fbabb3f8aca22efb5c2b9abec397a04b23a9c4 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:55:32 +0100 Subject: [PATCH 025/163] mason: share idle data timers and native FSEvents delivery Keep per-root data deadlines and existing work pools, while cancelling pending registrations on eviction. A shared two-worker release pool keeps cache destruction off the timer. Preserve per-lock heartbeat threads and transient-unbind watcher retention by parent decision. Native FSEvents streams share one serial delivery queue without changing per-root backend/filter queues. The stopped quiet-timer test now checks deadline cancellation and weak ownership rather than requiring a private waiting thread; its no-fire-after-stop assertion is retained. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 3161f0f06c464c4a38791a0e2421c9a0f4fa8f37) (cherry picked from commit 5f1f4509dc362176845e80fd34a2781f37ed5e4b) --- crates/aft/src/data_layer/runtime.rs | 383 +++++++++++++++------ crates/aft/src/inspect/manager.rs | 127 ++++++- crates/aft/src/lib.rs | 1 + crates/aft/src/shared_timer.rs | 201 +++++++++++ crates/aft/src/watcher_backend/fsevents.rs | 169 ++++++--- 5 files changed, 714 insertions(+), 167 deletions(-) create mode 100644 crates/aft/src/shared_timer.rs diff --git a/crates/aft/src/data_layer/runtime.rs b/crates/aft/src/data_layer/runtime.rs index a8b14d8e33..7030b5fb8b 100644 --- a/crates/aft/src/data_layer/runtime.rs +++ b/crates/aft/src/data_layer/runtime.rs @@ -327,9 +327,10 @@ impl Runner { /// When a failed build is tried again. #[derive(Default)] struct BuildRetry { - /// No build starts before this; a waiting thread starts one then. + /// No build starts before this; a shared deadline starts one then. at: Option, in_a_row: u32, + timer: Option, } /// A checkout and the state of its background work, shared by the work @@ -337,6 +338,7 @@ struct BuildRetry { struct Work { checkout: Arc, runner: Runner, + stopped: AtomicBool, /// A build is queued or running. building: Arc, /// A fold of the checkout's segments is queued or running. @@ -400,9 +402,12 @@ impl Work { /// Starts a build unless one is queued or running, or a failed one is /// waiting to be tried again. A build that fails is tried again after a /// wait that doubles each time, until one succeeds or the checkout is - /// dropped; the wait is a sleeping thread, never an executor worker. + /// dropped; the wait uses the shared timer, never an executor worker. /// Returns whether this call queued a build, so its cause is logged once. fn start_build(self: &Arc) -> bool { + if self.stopped.load(Ordering::Acquire) { + return false; + } { let retry = self .retry @@ -417,10 +422,13 @@ impl Work { } let flag = ClearOnDrop(Arc::clone(&self.building)); let Some(reservation) = self.checkout.reserve_build() else { - return; + return false; }; let work = Arc::clone(self); self.runner.run(BUILD_REQUEST, Pool::Build, move || { + if work.stopped.load(Ordering::Acquire) { + return; + } #[cfg(test)] while build_holds_for_test() .lock() @@ -467,15 +475,16 @@ impl Work { } fn retry_build_later(self: &Arc, error: &DataError) { - let wait = { + let (wait, at) = { let mut retry = self .retry .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); retry.in_a_row += 1; let wait = backoff(BUILD_RETRY, BUILD_RETRY_MAX, retry.in_a_row); - retry.at = Some(Instant::now() + wait); - wait + let at = Instant::now() + wait; + retry.at = Some(at); + (wait, at) }; crate::slog_warn!( "data layer build failed for {} (trying again in {} s): {error}", @@ -483,14 +492,20 @@ impl Work { wait.as_secs() ); let weak = Arc::downgrade(self); - let _ = std::thread::Builder::new() - .name("aft-data-build-retry".to_string()) - .spawn(move || { - std::thread::sleep(wait); - if let Some(work) = weak.upgrade() { - work.start_build(); - } - }); + let timer = crate::shared_timer::Timer::at(at, move || { + if let Some(work) = weak.upgrade() { + work.start_build(); + } + None + }) + .ok(); + let mut retry = self + .retry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !self.stopped.load(Ordering::Acquire) { + retry.timer = timer; + } } /// Resolves the call graph again until no recorded change is pending, @@ -499,6 +514,9 @@ impl Work { /// ending, by the check after it lets go of the flag. A refresh that /// finds the stored call graph damaged starts a build, which replaces it. fn start_refresh(self: &Arc, publish: Publish) { + if self.stopped.load(Ordering::Acquire) { + return; + } if !self.checkout.planes().callgraph { // Without a call graph there is nothing to resolve: the changes // laid over the text index are folded in by a text-only build, @@ -530,6 +548,9 @@ impl Work { // runs beside the root's reads and edits, one at a time per root // (the `refreshing` flag), in the refresh pool. self.runner.run(REFRESH_REQUEST, Pool::Refresh, move || { + if work.stopped.load(Ordering::Acquire) { + return; + } let mut flag = Some(flag); loop { let result = work.checkout.refresh_callgraph(); @@ -856,6 +877,9 @@ impl Work { /// them. A failed round is tried again after a wait that doubles each /// time, as a failed build is. fn start_fill(self: &Arc) { + if self.stopped.load(Ordering::Acquire) { + return; + } if !self.meaning.enabled() || self.meaning.reads_only() { return; } @@ -874,7 +898,7 @@ impl Work { let flag = ClearOnDrop(Arc::clone(&self.filling)); let work = Arc::clone(self); self.runner.run(FILL_REQUEST, Pool::Fill, move || { - if crate::executor::current_job_cancelled() { + if work.stopped.load(Ordering::Acquire) || crate::executor::current_job_cancelled() { // Runs again when the root is bound and a search asks. return; } @@ -904,15 +928,16 @@ impl Work { } fn retry_fill_later(self: &Arc, error: &str) { - let wait = { + let (wait, at) = { let mut retry = self .fill_retry .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); retry.in_a_row += 1; let wait = backoff(BUILD_RETRY, BUILD_RETRY_MAX, retry.in_a_row); - retry.at = Some(Instant::now() + wait); - wait + let at = Instant::now() + wait; + retry.at = Some(at); + (wait, at) }; crate::slog_warn!( "data layer meaning fill failed for {} (trying again in {} s): {error}", @@ -920,14 +945,20 @@ impl Work { wait.as_secs() ); let weak = Arc::downgrade(self); - let _ = std::thread::Builder::new() - .name("aft-data-fill-retry".to_string()) - .spawn(move || { - std::thread::sleep(wait); - if let Some(work) = weak.upgrade() { - work.start_fill(); - } - }); + let timer = crate::shared_timer::Timer::at(at, move || { + if let Some(work) = weak.upgrade() { + work.start_fill(); + } + None + }) + .ok(); + let mut retry = self + .fill_retry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !self.stopped.load(Ordering::Acquire) { + retry.timer = timer; + } } /// Tells the status listener, if any, that the checkout's status may @@ -967,11 +998,24 @@ pub(crate) struct DataCheckout { } impl Drop for DataCheckout { - // An evicted checkout must not leave its quiet-timer thread asleep for - // up to `QUIET`: its refresh would find nothing to refresh anyway. + // An evicted checkout must not dispatch delayed work even if an in-flight + // pool job still holds Work alive. fn drop(&mut self) { + self.work.stopped.store(true, Ordering::Release); self.quiet.stop(); self.work.settle.stop(); + self.work + .retry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .timer + .take(); + self.work + .fill_retry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .timer + .take(); } } @@ -1079,6 +1123,7 @@ impl DataCheckout { } }; let work = Arc::new_cyclic(|weak: &Weak| Work { + stopped: AtomicBool::new(false), runner: Runner { root: checkout.root().to_path_buf(), executor: parking_lot::Mutex::new(executor), @@ -1223,9 +1268,8 @@ impl DataCheckout { } /// Calls `fire` once a checkout has gone `quiet` without a recorded change -/// (for the call graph, [`QUIET`], to refresh it). One timer thread runs per -/// checkout at most, and only while a change is waiting: each change moves -/// the deadline on, and the thread sleeps until the latest one. +/// (for the call graph, [`QUIET`], to refresh it). Pending checkouts share the +/// process timer; each change moves the deadline checked at its next wake. struct QuietTimer { quiet: Duration, /// When the last change was recorded, in milliseconds since `epoch`. @@ -1233,10 +1277,9 @@ struct QuietTimer { armed: AtomicBool, epoch: Instant, fire: Box, - /// Set when the checkout is dropped: the waiting thread ends at once - /// instead of sleeping out its deadline. + /// Also fences dispatch against checkout shutdown. stopped: std::sync::Mutex, - wake: std::sync::Condvar, + timer: std::sync::Mutex>, #[cfg(feature = "test-timing-hooks")] test_gate: bool, } @@ -1250,33 +1293,22 @@ impl QuietTimer { epoch: Instant::now(), fire: Box::new(fire), stopped: std::sync::Mutex::new(false), - wake: std::sync::Condvar::new(), + timer: std::sync::Mutex::new(None), #[cfg(feature = "test-timing-hooks")] test_gate: false, } } - /// Ends the waiting thread, if any, without firing; later changes start - /// none. + /// Removes the pending callback without firing; later changes arm none. fn stop(&self) { *self .stopped .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = true; - self.wake.notify_all(); - } - - /// Sleeps up to `duration`, or until [`Self::stop`]; true once stopped. - fn sleep(&self, duration: Duration) -> bool { - let stopped = self - .stopped + self.timer .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let (stopped, _) = self - .wake - .wait_timeout_while(stopped, duration, |stopped| !*stopped) - .unwrap_or_else(std::sync::PoisonError::into_inner); - *stopped + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); } fn now_ms(&self) -> u64 { @@ -1293,7 +1325,7 @@ impl QuietTimer { /// Changes continue moving its deadline while the integration test writes /// and removes a burst; the deadline is checked again after release. #[cfg(feature = "test-timing-hooks")] - fn hold_for_test(&self) -> Option { + fn gate_for_test(&self) -> Option { if !self.test_gate { return None; } @@ -1302,16 +1334,10 @@ impl QuietTimer { return None; } std::fs::write(gate.with_extension("waiting"), b"waiting").expect("signal quiet gate"); - while gate.exists() && !gate.with_extension("release").exists() { - if self.sleep(Duration::from_millis(10)) { - return None; - } - } Some(gate) } - /// Notes a recorded change, and starts the timer thread unless one is - /// waiting already. + /// Notes a recorded change, and arms a deadline unless one is pending. fn changed(self: &Arc) { self.note_change(); self.arm(); @@ -1331,54 +1357,79 @@ impl QuietTimer { /// Wait from the last recorded change, not from the end of its refresh. fn arm(self: &Arc) { - if self.sleep(Duration::ZERO) || self.armed.swap(true, Ordering::AcqRel) { + let stopped = self + .stopped + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if *stopped || self.armed.swap(true, Ordering::AcqRel) { return; } - let timer = Arc::clone(self); + let timer = Arc::downgrade(self); let quiet_ms = u64::try_from(self.quiet.as_millis()).unwrap_or(u64::MAX); - let spawned = std::thread::Builder::new() - .name("aft-data-quiet".to_string()) - .spawn(move || { - #[cfg(feature = "test-timing-hooks")] - let gate = timer.hold_for_test(); - if timer.sleep(Duration::ZERO) { - return; + #[cfg(feature = "test-timing-hooks")] + let mut gate = None; + #[cfg(feature = "test-timing-hooks")] + let mut gate_checked = false; + let scheduled = crate::shared_timer::Timer::at(Instant::now(), move || { + let timer = timer.upgrade()?; + let stopped = timer + .stopped + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if *stopped { + return None; + } + #[cfg(feature = "test-timing-hooks")] + { + if !gate_checked { + gate = timer.gate_for_test(); + gate_checked = true; } - loop { - let due = timer - .last_change_ms - .load(Ordering::Acquire) - .saturating_add(quiet_ms); - let now = timer.now_ms(); - if now < due { - if timer.sleep(Duration::from_millis(due - now)) { - break; - } - continue; - } - timer.armed.store(false, Ordering::Release); - // A change recorded between the deadline check and letting - // go of the flag found the timer armed and started none; if - // there was one, this thread keeps waiting for it. - if timer - .last_change_ms - .load(Ordering::Acquire) - .saturating_add(quiet_ms) - > timer.now_ms() - && !timer.armed.swap(true, Ordering::AcqRel) - { - continue; - } - (timer.fire)(); - #[cfg(feature = "test-timing-hooks")] - if let Some(gate) = &gate { - std::fs::write(gate.with_extension("settled"), b"settled") - .expect("signal quiet fire"); - } - break; + if gate + .as_ref() + .is_some_and(|gate| gate.exists() && !gate.with_extension("release").exists()) + { + return Some(Instant::now() + Duration::from_millis(10)); } - }); - if spawned.is_err() { + } + let due = timer + .last_change_ms + .load(Ordering::Acquire) + .saturating_add(quiet_ms); + let now = timer.now_ms(); + if now < due { + return Some(Instant::now() + Duration::from_millis(due - now)); + } + timer.armed.store(false, Ordering::Release); + // A change recorded between the deadline check and letting + // go of the flag found the timer armed and started none; if + // there was one, keep the deadline armed for it. + if timer + .last_change_ms + .load(Ordering::Acquire) + .saturating_add(quiet_ms) + > timer.now_ms() + && !timer.armed.swap(true, Ordering::AcqRel) + { + return Some(Instant::now()); + } + // The callback only admits/enqueues work; refreshes, builds and + // fills still execute in their existing data pools. + drop(stopped); + (timer.fire)(); + #[cfg(feature = "test-timing-hooks")] + if let Some(gate) = &gate { + std::fs::write(gate.with_extension("settled"), b"settled") + .expect("signal quiet fire"); + } + None + }); + if let Ok(timer) = scheduled { + *self + .timer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(timer); + } else { self.armed.store(false, Ordering::Release); } } @@ -1793,6 +1844,114 @@ mod tests { } } + #[cfg(target_os = "linux")] + #[test] + fn binding_many_roots_shares_waiting_threads_and_eviction_removes_deadlines() { + use crate::shared_timer::Timer; + const ROOTS: usize = 12; + let mut fixtures = Vec::new(); + let mut managers = Vec::new(); + let mut works = Vec::new(); + for _ in 0..ROOTS { + let fixture = PublicationFixture::new(QUIET, QUIET); + fixture.handle.quiet.changed(); + fixture.handle.work.settle.changed(); + // Keep real retry registrations pending while all roots are bound. + fixture.handle.work.retry.lock().unwrap().in_a_row = 10; + fixture.handle.work.fill_retry.lock().unwrap().in_a_row = 10; + fixture.handle.work.retry_build_later(&DataError::Cancelled); + fixture + .handle + .work + .retry_fill_later("retry registration test"); + let manager = Arc::new(crate::inspect::InspectManager::new()); + manager.note_dead_code_request(Instant::now()); + works.push(Arc::clone(&fixture.handle.work)); + managers.push(manager); + fixtures.push(fixture); + } + // Initial checks run immediately, then register the actual idle deadline. + wait_until(|| { + managers.iter().all(|manager| { + manager + .dead_code_timer_registration_for_test() + .and_then(Timer::deadline_for_test) + .is_some_and(|at| at > Instant::now() + Duration::from_secs(60)) + }) + }); + let mut registrations = Vec::new(); + for (fixture, manager) in fixtures.iter().zip(&managers) { + for timer in [&fixture.handle.quiet, &fixture.handle.work.settle] { + registrations.push( + timer + .timer + .lock() + .unwrap() + .as_ref() + .unwrap() + .registration_for_test(), + ); + } + for retry in [&fixture.handle.work.retry, &fixture.handle.work.fill_retry] { + registrations.push( + retry + .lock() + .unwrap() + .timer + .as_ref() + .unwrap() + .registration_for_test(), + ); + } + registrations.push(manager.dead_code_timer_registration_for_test().unwrap()); + } + wait_until(|| { + registrations + .iter() + .all(|id| Timer::registered_for_test(*id)) + }); + let names = std::fs::read_dir("/proc/self/task") + .unwrap() + .map(|entry| { + std::fs::read_to_string(entry.unwrap().path().join("comm")).unwrap_or_default() + }) + .collect::>(); + let count = |name: &str| names.iter().filter(|comm| comm.trim_end() == name).count(); + // Linux comm truncates thread names to fifteen bytes. + assert_eq!( + count("aft-shared-time"), + 1, + "one deadline service for every root" + ); + assert_eq!( + count("aft-dead-code-r"), + 2, + "release workers grew with roots" + ); + assert_eq!(count("aft-data-quiet"), 0); + assert_eq!(count("aft-data-fill-r"), 0); + assert_eq!(count("aft-data-build-"), 0); + + let quiet = Arc::downgrade(&fixtures[0].handle.quiet); + let manager = Arc::downgrade(&managers[0]); + drop(fixtures); + drop(managers); + wait_until(|| quiet.upgrade().is_none() && manager.upgrade().is_none()); + assert!( + registrations + .iter() + .all(|id| !Timer::registered_for_test(*id)), + "an evicted root left a timer registered" + ); + // Even a pool job retaining Work cannot rearm an evicted checkout. + for work in &works { + work.retry_build_later(&DataError::Cancelled); + work.retry_fill_later("evicted root"); + assert!(work.retry.lock().unwrap().timer.is_none()); + assert!(work.fill_retry.lock().unwrap().timer.is_none()); + } + } + #[test] fn the_next_edit_gets_a_fresh_view_before_a_slow_publication() { let fixture = PublicationFixture::new(Duration::from_secs(60), QUIET); @@ -2301,22 +2460,34 @@ mod tests { } #[test] - fn a_stopped_quiet_timer_ends_its_thread_without_firing() { + fn a_stopped_quiet_timer_cancels_its_deadline_without_firing() { let fired = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&fired); let timer = Arc::new(QuietTimer::new(Duration::from_secs(120), move || { count.fetch_add(1, Ordering::SeqCst); })); timer.changed(); - // The waiting thread holds the timer. - assert_eq!(Arc::strong_count(&timer), 2); + let registration = timer + .timer + .lock() + .unwrap() + .as_ref() + .unwrap() + .registration_for_test(); timer.stop(); let deadline = Instant::now() + Duration::from_secs(10); while Arc::strong_count(&timer) > 1 && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); } - assert_eq!(Arc::strong_count(&timer), 1, "the thread is still waiting"); - // Changes after the stop start no thread. + assert_eq!( + Arc::strong_count(&timer), + 1, + "the callback retained its owner" + ); + assert!(!crate::shared_timer::Timer::registered_for_test( + registration + )); + // Changes after the stop register no deadline. timer.changed(); assert_eq!(Arc::strong_count(&timer), 1); assert_eq!(fired.load(Ordering::SeqCst), 0); diff --git a/crates/aft/src/inspect/manager.rs b/crates/aft/src/inspect/manager.rs index 0172c7a73c..3846b71513 100644 --- a/crates/aft/src/inspect/manager.rs +++ b/crates/aft/src/inspect/manager.rs @@ -704,8 +704,11 @@ pub struct InspectManager { /// When an `aft_inspect` that read the dead-code graph last finished; /// `None` once the graph was released, or before any such request. last_dead_code_request: Mutex>, - /// Set while a thread is waiting to release the dead-code graph. + /// Set while a deadline or release-pool job is pending. dead_code_release_armed: AtomicBool, + dead_code_release_timer: Mutex>, + dead_code_release_generation: AtomicU64, + dead_code_release_stopped: AtomicBool, /// How many dead-code passes are waiting for their call graph right now: /// for it to be published, or to catch up with recorded edits. A pass /// that is building but not waiting is working out dead code itself. @@ -875,6 +878,9 @@ impl InspectManager { data_layer_dead_code: crate::data_layer::tools::inspect::DeadCodeSources::default(), last_dead_code_request: Mutex::new(None), dead_code_release_armed: AtomicBool::new(false), + dead_code_release_timer: Mutex::new(None), + dead_code_release_generation: AtomicU64::new(0), + dead_code_release_stopped: AtomicBool::new(false), dead_code_callgraph_waits: AtomicUsize::new(0), #[cfg(test)] interactive_tier2_acquisitions: AtomicU64::new(0), @@ -923,6 +929,15 @@ impl InspectManager { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .request_cancel(); + let mut timer = self + .dead_code_release_timer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.dead_code_release_stopped.store(true, Ordering::SeqCst); + self.dead_code_release_generation + .fetch_add(1, Ordering::SeqCst); + timer.take(); + self.dead_code_release_armed.store(false, Ordering::SeqCst); } pub(crate) fn set_root_lifecycle(&self, lifecycle: crate::context::SubcLifecycleAdmission) { @@ -934,6 +949,8 @@ impl InspectManager { } pub(crate) fn resume_root_work(&self) { + self.dead_code_release_stopped + .store(false, Ordering::SeqCst); let mut token = self .root_work_cancellation .lock() @@ -1771,51 +1788,104 @@ impl InspectManager { } /// Note that an `aft_inspect` which read the dead-code graph finished at - /// `now`, and make sure a thread will release the graph once it has gone + /// `now`, and schedule release of the graph once it has gone /// [`DEAD_CODE_GRAPH_IDLE_RELEASE`] without another such request. pub(crate) fn note_dead_code_request(self: &Arc, now: Instant) { + if self.dead_code_release_stopped.load(Ordering::SeqCst) { + return; + } self.record_dead_code_request(now); if self.dead_code_release_armed.swap(true, Ordering::SeqCst) { return; } + let generation = self.dead_code_release_generation.load(Ordering::SeqCst); + self.schedule_dead_code_release(Instant::now(), generation); + } + + fn schedule_dead_code_release(self: &Arc, at: Instant, generation: u64) { + // Cache destruction must not block the timer or the interactive inspect + // pool. Two release workers bound concurrency across all roots. + static POOL: std::sync::OnceLock> = std::sync::OnceLock::new(); + let pool = POOL.get_or_init(|| { + rayon::ThreadPoolBuilder::new() + .num_threads(2) + .thread_name(|_| "aft-dead-code-release".into()) + .build() + .ok() + }); + let mut timer = self + .dead_code_release_timer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.dead_code_release_stopped.load(Ordering::SeqCst) { + self.dead_code_release_armed.store(false, Ordering::SeqCst); + return; + } + if self.dead_code_release_generation.load(Ordering::SeqCst) != generation { + return; + } + let Some(pool) = pool else { + self.dead_code_release_armed.store(false, Ordering::SeqCst); + return; + }; let manager = Arc::downgrade(self); - let spawned = std::thread::Builder::new() - .name("aft-dead-code-release".into()) - .spawn(move || loop { - // Hold the manager only while checking, so dropping the - // manager ends the thread at its next wake-up. + *timer = crate::shared_timer::Timer::at(at, move || { + let manager = manager.clone(); + pool.spawn(move || { let Some(current) = manager.upgrade() else { return; }; + if current.dead_code_release_generation.load(Ordering::SeqCst) != generation { + return; + } match current.release_idle_dead_code_graph(Instant::now()) { DeadCodeGraphRelease::KeepFor(wait) => { - drop(current); - std::thread::sleep(wait); + current.schedule_dead_code_release(Instant::now() + wait, generation); } DeadCodeGraphRelease::Released | DeadCodeGraphRelease::NotHeld => { + let timer = current + .dead_code_release_timer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.dead_code_release_generation.load(Ordering::SeqCst) != generation + { + return; + } current .dead_code_release_armed .store(false, Ordering::SeqCst); - // A request noted while this thread was deciding found - // it armed and started none; keep watching for it. let pending = current .last_dead_code_request .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .is_some(); - if !pending || current.dead_code_release_armed.swap(true, Ordering::SeqCst) - { - return; + let rearm = pending + && !current.dead_code_release_armed.swap(true, Ordering::SeqCst); + drop(timer); + if rearm { + current.schedule_dead_code_release(Instant::now(), generation); } } } }); - if spawned.is_err() { + None + }) + .ok(); + if timer.is_none() { self.dead_code_release_armed.store(false, Ordering::SeqCst); } } - /// Record a dead-code request at `now` without arming the release thread. + #[cfg(test)] + pub(crate) fn dead_code_timer_registration_for_test(&self) -> Option { + self.dead_code_release_timer + .lock() + .unwrap() + .as_ref() + .map(crate::shared_timer::Timer::registration_for_test) + } + + /// Record a dead-code request at `now` without arming a release deadline. pub(crate) fn record_dead_code_request(&self, now: Instant) { *self .last_dead_code_request @@ -6594,6 +6664,31 @@ mod guard_tests { ); } + #[test] + fn dead_code_release_cancellation_removes_deadline_and_resumes() { + let manager = Arc::new(InspectManager::new()); + manager + .dead_code_release_armed + .store(true, Ordering::SeqCst); + manager.schedule_dead_code_release(Instant::now() + DEAD_CODE_GRAPH_IDLE_RELEASE, 0); + let id = manager.dead_code_timer_registration_for_test().unwrap(); + assert!(crate::shared_timer::Timer::registered_for_test(id)); + manager.cancel_root_work(); + assert!(!crate::shared_timer::Timer::registered_for_test(id)); + assert!(!manager.dead_code_release_armed.load(Ordering::SeqCst)); + // A completion already queued before teardown cannot rearm its deadline. + manager.schedule_dead_code_release(Instant::now(), 0); + assert!(manager.dead_code_timer_registration_for_test().is_none()); + manager.resume_root_work(); + let generation = manager.dead_code_release_generation.load(Ordering::SeqCst); + manager + .schedule_dead_code_release(Instant::now() + DEAD_CODE_GRAPH_IDLE_RELEASE, generation); + let id = manager.dead_code_timer_registration_for_test().unwrap(); + assert!(crate::shared_timer::Timer::registered_for_test(id)); + drop(manager); + assert!(!crate::shared_timer::Timer::registered_for_test(id)); + } + /// The dead-code graph a request builds stays in memory for /// `DEAD_CODE_GRAPH_IDLE_RELEASE` after the last dead-code request, is /// released then, and is built again by the next request. The clock is diff --git a/crates/aft/src/lib.rs b/crates/aft/src/lib.rs index c4228830db..d77379cdf8 100644 --- a/crates/aft/src/lib.rs +++ b/crates/aft/src/lib.rs @@ -197,6 +197,7 @@ pub mod secret_proxy; pub mod semantic_admission; pub mod semantic_index; pub mod setup_plan; +pub(crate) mod shared_timer; pub(crate) mod stage_timer; pub mod standing_roots; pub(crate) mod status_push; diff --git a/crates/aft/src/shared_timer.rs b/crates/aft/src/shared_timer.rs new file mode 100644 index 0000000000..848cc98fcb --- /dev/null +++ b/crates/aft/src/shared_timer.rs @@ -0,0 +1,201 @@ +//! Process-wide deadlines. Callbacks only check state or enqueue work: blocking +//! I/O and cache destruction belong on the caller's existing work pool. + +use std::collections::{BTreeMap, HashMap}; +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex, OnceLock}; +use std::time::Instant; + +type Callback = Box Option + Send>; +type Key = (Instant, u64); + +struct Entry { + active: Arc, + fire: Callback, +} + +#[derive(Default)] +struct Queue { + next_id: u64, + deadlines: BTreeMap, + keys: HashMap, +} + +#[derive(Default)] +struct Service { + queue: Mutex, + wake: Condvar, +} + +fn service() -> io::Result<&'static Arc> { + static SERVICE: OnceLock, String>> = OnceLock::new(); + SERVICE + .get_or_init(|| { + let service = Arc::new(Service::default()); + let worker = Arc::clone(&service); + std::thread::Builder::new() + .name("aft-shared-timer".into()) + .spawn(move || worker.run()) + .map_err(|error| error.to_string())?; + Ok(service) + }) + .as_ref() + .map_err(|error| io::Error::other(error.clone())) +} + +impl Service { + fn run(&self) { + loop { + let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + let Some((&(at, id), _)) = queue.deadlines.first_key_value() else { + drop(self.wake.wait(queue).unwrap_or_else(|e| e.into_inner())); + continue; + }; + let now = Instant::now(); + if now < at { + drop( + self.wake + .wait_timeout(queue, at - now) + .unwrap_or_else(|e| e.into_inner()), + ); + continue; + } + let mut entry = queue.deadlines.remove(&(at, id)).unwrap(); + queue.keys.remove(&id); + drop(queue); + if !entry.active.load(Ordering::Acquire) { + continue; + } + // A bad callback must not disable every root's deadlines. + let next = std::panic::catch_unwind(std::panic::AssertUnwindSafe(&mut entry.fire)); + if let Ok(Some(at)) = next { + let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + if entry.active.load(Ordering::Acquire) { + queue.keys.insert(id, (at, id)); + queue.deadlines.insert((at, id), entry); + } + } + } + } +} + +/// Cancels a pending deadline on drop, removing its callback immediately. A +/// callback already being dispatched must also fence its work with the root's +/// lifecycle (usually a weak owner and the executor's cancellation token). +pub(crate) struct Timer { + service: &'static Arc, + id: u64, + active: Arc, +} + +impl Timer { + #[cfg(test)] + pub(crate) fn registration_for_test(&self) -> u64 { + self.id + } + + #[cfg(test)] + pub(crate) fn registered_for_test(id: u64) -> bool { + service() + .unwrap() + .queue + .lock() + .unwrap() + .keys + .contains_key(&id) + } + + #[cfg(all(test, target_os = "linux"))] + pub(crate) fn deadline_for_test(id: u64) -> Option { + service() + .unwrap() + .queue + .lock() + .unwrap() + .keys + .get(&id) + .map(|key| key.0) + } + + pub(crate) fn at( + at: Instant, + fire: impl FnMut() -> Option + Send + 'static, + ) -> io::Result { + let service = service()?; + let active = Arc::new(AtomicBool::new(true)); + let mut queue = service.queue.lock().unwrap_or_else(|e| e.into_inner()); + let id = queue.next_id; + queue.next_id = queue.next_id.checked_add(1).expect("timer id exhausted"); + queue.keys.insert(id, (at, id)); + queue.deadlines.insert( + (at, id), + Entry { + active: Arc::clone(&active), + fire: Box::new(fire), + }, + ); + service.wake.notify_one(); + Ok(Self { + service, + id, + active, + }) + } +} + +impl Drop for Timer { + fn drop(&mut self) { + self.active.store(false, Ordering::Release); + let mut queue = self.service.queue.lock().unwrap_or_else(|e| e.into_inner()); + let removed = queue + .keys + .remove(&self.id) + .and_then(|key| queue.deadlines.remove(&key)); + self.service.wake.notify_one(); + drop(queue); + drop(removed); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::mpsc; + use std::time::Duration; + + #[test] + fn deadlines_rearm_cancel_and_survive_panics() { + let (tx, rx) = mpsc::channel(); + let start = Instant::now(); + let mut count = 0; + let _timer = Timer::at(start, move || { + count += 1; + tx.send((count, Instant::now())).unwrap(); + (count == 1).then_some(start + Duration::from_millis(30)) + }) + .unwrap(); + assert_eq!(rx.recv_timeout(Duration::from_secs(2)).unwrap().0, 1); + let (count, at) = rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(count, 2); + assert!(at >= start + Duration::from_millis(30)); + + let (tx, rx) = mpsc::channel(); + let timer = Timer::at(Instant::now() + Duration::from_millis(30), move || { + tx.send(()).unwrap(); + None + }) + .unwrap(); + drop(timer); + assert!(rx.recv_timeout(Duration::from_millis(60)).is_err()); + + let _panic = Timer::at(Instant::now(), || panic!("isolated callback panic")).unwrap(); + let (tx, rx) = mpsc::channel(); + let _later = Timer::at(Instant::now() + Duration::from_millis(10), move || { + tx.send(()).unwrap(); + None + }) + .unwrap(); + rx.recv_timeout(Duration::from_secs(2)).unwrap(); + } +} diff --git a/crates/aft/src/watcher_backend/fsevents.rs b/crates/aft/src/watcher_backend/fsevents.rs index 8d9d8eabca..c10446a3b3 100644 --- a/crates/aft/src/watcher_backend/fsevents.rs +++ b/crates/aft/src/watcher_backend/fsevents.rs @@ -3,7 +3,7 @@ use std::os::unix::ffi::OsStringExt; use std::path::{Path, PathBuf}; use std::ptr; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{mpsc, Arc}; +use std::sync::{mpsc, Arc, OnceLock}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; @@ -459,11 +459,66 @@ impl DirectoryWatcher { struct FsEventsStream { stream: usize, - run_loop: usize, - join: Option>, + queue: usize, sender: NativeEventSender, } +/// All streams deliver on one serial queue. Only native event translation runs +/// here; topology probes, exclusion replanning and per-root backpressure remain +/// on their existing workers. The queue lives for the process, not for a root. +fn delivery_queue() -> notify::Result { + static QUEUE: OnceLock = OnceLock::new(); + let queue = *QUEUE.get_or_init(|| unsafe { + dispatch_queue_create(c"aft-fsevents-delivery".as_ptr(), ptr::null()) as usize + }); + if queue == 0 { + Err(notify::Error::generic( + "failed to create FSEvents delivery queue", + )) + } else { + Ok(queue) + } +} + +struct StreamAction { + stream: usize, + start: bool, + started: bool, +} + +extern "C" fn act_on_stream(context: *mut libc::c_void) { + unsafe { + let action = &mut *context.cast::(); + let stream = action.stream as fs::FSEventStreamRef; + if action.start { + action.started = fs::FSEventStreamStart(stream) != 0; + } else { + fs::FSEventStreamStop(stream); + fs::FSEventStreamInvalidate(stream); + fs::FSEventStreamRelease(stream); + } + } +} + +fn stream_action(queue: usize, stream: usize, start: bool) -> bool { + let mut action = StreamAction { + stream, + start, + started: false, + }; + // Synchronous execution on the delivery queue fences callbacks: after a + // stop returns no callback can still be using the released stream context. + // Stream owners are backend/config workers, never native callbacks. + unsafe { + dispatch_sync_f( + queue as *mut _, + (&mut action as *mut StreamAction).cast(), + act_on_stream, + ); + } + action.started +} + impl FsEventsStream { #[cfg(test)] fn start( @@ -579,49 +634,28 @@ impl FsEventsStream { )); } - let stream_address = stream as usize; - let (run_loop_tx, run_loop_rx) = mpsc::sync_channel(1); - let join = match thread::Builder::new() - .name("aft-fsevents-runloop".to_string()) - .spawn(move || { - let stream = stream_address as fs::FSEventStreamRef; - unsafe { - let run_loop = cf::CFRunLoopGetCurrent(); - fs::FSEventStreamScheduleWithRunLoop( - stream, - run_loop, - cf::kCFRunLoopDefaultMode, - ); - if fs::FSEventStreamStart(stream) == 0 { - let _ = run_loop_tx.send(0); - } else { - let _ = run_loop_tx.send(run_loop as usize); - cf::CFRunLoopRun(); - } - fs::FSEventStreamStop(stream); - fs::FSEventStreamInvalidate(stream); - fs::FSEventStreamRelease(stream); - } - }) { - Ok(join) => join, + let queue = match delivery_queue() { + Ok(queue) => queue, Err(error) => { unsafe { fs::FSEventStreamInvalidate(stream); fs::FSEventStreamRelease(stream); } - return Err(notify::Error::io(error)); + return Err(error); } }; - let run_loop = run_loop_rx.recv().map_err(notify::Error::from)?; - if run_loop == 0 { - let _ = join.join(); + unsafe { + FSEventStreamSetDispatchQueue(stream, queue as *mut _); + } + let stream_address = stream as usize; + if !stream_action(queue, stream_address, true) { + stream_action(queue, stream_address, false); return Err(notify::Error::generic("FSEventStreamStart failed")); } Ok(Self { stream: stream_address, - run_loop, - join: Some(join), + queue, sender, }) } @@ -633,16 +667,7 @@ impl FsEventsStream { impl Drop for FsEventsStream { fn drop(&mut self) { - let run_loop = self.run_loop as cf::CFRunLoopRef; - unsafe { - while CFRunLoopIsWaiting(run_loop) == 0 { - thread::yield_now(); - } - cf::CFRunLoopStop(run_loop); - } - if let Some(join) = self.join.take() { - let _ = join.join(); - } + stream_action(self.queue, self.stream, false); } } @@ -884,7 +909,16 @@ fn create_cf_path_array(paths: &[PathBuf]) -> notify::Result cf::Boolean; + fn FSEventStreamSetDispatchQueue(stream: fs::FSEventStreamRef, queue: *mut libc::c_void); + fn dispatch_queue_create( + label: *const libc::c_char, + attr: *const libc::c_void, + ) -> *mut libc::c_void; + fn dispatch_sync_f( + queue: *mut libc::c_void, + context: *mut libc::c_void, + work: extern "C" fn(*mut libc::c_void), + ); } #[cfg(test)] @@ -1007,6 +1041,51 @@ mod tests { } } + #[test] + fn streams_share_delivery_queue_and_drop_independently() { + let roots = [tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()]; + let mut streams = Vec::new(); + let mut receivers = Vec::new(); + let mut paths = Vec::new(); + for root in &roots { + let root = std::fs::canonicalize(root.path()).unwrap(); + let (tx, rx) = mpsc::channel(); + let stream = FsEventsStream::start( + &root, + &[], + tx, + fs::kFSEventStreamEventIdSinceNow, + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + assert_eq!(stream.queue, delivery_queue().unwrap()); + paths.push(root.join("before-drop.rs")); + streams.push(stream); + receivers.push(rx); + } + for (path, rx) in paths.iter().zip(&receivers) { + std::fs::write(path, b"pub fn before() {}\n").unwrap(); + wait_for_raw_path(rx, path); + } + drop(streams.remove(0)); + // Invalidation and release on the shared queue release the stream's + // callback context, so no sender or stream registration survives eviction. + while receivers[0].try_recv().is_ok() {} + assert!(matches!( + receivers[0].try_recv(), + Err(mpsc::TryRecvError::Disconnected) + )); + let survivor = paths[1].with_file_name("after-drop.rs"); + std::fs::write(&survivor, b"pub fn after() {}\n").unwrap(); + wait_for_raw_path(&receivers[1], &survivor); + drop(streams); + while receivers[1].try_recv().is_ok() {} + assert!(matches!( + receivers[1].try_recv(), + Err(mpsc::TryRecvError::Disconnected) + )); + } + fn symlink_replacement_fixture( starts_as_directory: bool, ) -> ( From 0d7c939c3f9bd5aa679154a9a412c77d44be578f Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:00:13 +0100 Subject: [PATCH 026/163] mason: instrument native FSEvents context release Add per-stream test-only release_context counters and native eventful/eventless drop probes with a five-second bound. Keep the original immediate-disconnection assertion unchanged until the probes identify which sender remains owned. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit f3e34cfb0f9b0a3816751b3748265638c4addfc4) (cherry picked from commit 13d18c39848e867a212fffce26297f1d62a52416) --- crates/aft/src/watcher_backend/fsevents.rs | 82 +++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/crates/aft/src/watcher_backend/fsevents.rs b/crates/aft/src/watcher_backend/fsevents.rs index c10446a3b3..94642b9dc3 100644 --- a/crates/aft/src/watcher_backend/fsevents.rs +++ b/crates/aft/src/watcher_backend/fsevents.rs @@ -461,6 +461,8 @@ struct FsEventsStream { stream: usize, queue: usize, sender: NativeEventSender, + #[cfg(test)] + context_release_calls: Arc, } /// All streams deliver on one serial queue. Only native event translation runs @@ -565,8 +567,12 @@ impl FsEventsStream { counters: Arc, ) -> notify::Result { let watched_paths = create_cf_path_array(paths)?; + #[cfg(test)] + let context_release_calls = Arc::new(AtomicU64::new(0)); let context_info = Box::into_raw(Box::new(CallbackContext { sender: sender.clone(), + #[cfg(test)] + release_calls: Arc::clone(&context_release_calls), last_delivered_id, nonrecursive_paths, #[cfg(test)] @@ -657,6 +663,8 @@ impl FsEventsStream { stream: stream_address, queue, sender, + #[cfg(test)] + context_release_calls, }) } @@ -673,6 +681,8 @@ impl Drop for FsEventsStream { struct CallbackContext { sender: NativeEventSender, + #[cfg(test)] + release_calls: Arc, last_delivered_id: Arc, nonrecursive_paths: Option>, #[cfg(test)] @@ -681,7 +691,10 @@ struct CallbackContext { extern "C" fn release_context(info: *const libc::c_void) { unsafe { - drop(Box::from_raw(info as *mut CallbackContext)); + let context = Box::from_raw(info as *mut CallbackContext); + #[cfg(test)] + context.release_calls.fetch_add(1, Ordering::SeqCst); + drop(context); } } @@ -1041,6 +1054,70 @@ mod tests { } } + fn wait_for_context_release(rx: &mpsc::Receiver>, releases: &AtomicU64) { + let start = Instant::now(); + let deadline = start + Duration::from_secs(5); + loop { + match rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(_) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => panic!( + "stream sender retained after five seconds; release_context calls={}", + releases.load(Ordering::SeqCst) + ), + } + } + assert_eq!( + releases.load(Ordering::SeqCst), + 1, + "release_context must run exactly once per stream" + ); + eprintln!( + "stream disconnected after {:?}; release_context calls=1", + start.elapsed() + ); + } + + fn prove_context_release_after_drop(deliver_events: bool) { + let root = tempfile::tempdir().unwrap(); + let root = std::fs::canonicalize(root.path()).unwrap(); + let (tx, rx) = mpsc::channel(); + let stream = FsEventsStream::start( + &root, + &[], + tx, + fs::kFSEventStreamEventIdSinceNow, + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + let releases = Arc::clone(&stream.context_release_calls); + if deliver_events { + let path = root.join("before-drop.rs"); + std::fs::write(&path, b"pub fn before() {}\n").unwrap(); + wait_for_raw_path(&rx, &path); + } else { + assert!(matches!(rx.try_recv(), Err(mpsc::TryRecvError::Empty))); + } + assert_eq!(releases.load(Ordering::SeqCst), 0); + drop(stream); + eprintln!( + "after drop: delivered_events={deliver_events}, release_context calls={}, channel={:?}", + releases.load(Ordering::SeqCst), + rx.try_recv() + ); + wait_for_context_release(&rx, &releases); + } + + #[test] + fn stream_context_released_once_after_eventful_drop() { + prove_context_release_after_drop(true); + } + + #[test] + fn stream_context_released_once_after_eventless_drop() { + prove_context_release_after_drop(false); + } + #[test] fn streams_share_delivery_queue_and_drop_independently() { let roots = [tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()]; @@ -1399,6 +1476,7 @@ mod tests { // Each acknowledged rescan republishes an unchanged matcher, as a real // overflow drain does, exercising the replacement-stream decision too. let mut context = CallbackContext { + release_calls: Arc::new(AtomicU64::new(0)), sender: NativeEventSender::Events(sender), last_delivered_id: Arc::new(AtomicU64::new(0)), nonrecursive_paths: None, @@ -1518,6 +1596,7 @@ mod tests { let (sender, rx) = mpsc::channel(); let cursor = Arc::new(AtomicU64::new(0)); let mut context = CallbackContext { + release_calls: Arc::new(AtomicU64::new(0)), sender: NativeEventSender::Batches(sender), last_delivered_id: Arc::clone(&cursor), nonrecursive_paths: None, @@ -1601,6 +1680,7 @@ mod tests { let root = std::fs::canonicalize(temp.path()).unwrap(); let (sender, rx) = mpsc::channel(); let mut context = CallbackContext { + release_calls: Arc::new(AtomicU64::new(0)), sender: NativeEventSender::Events(sender), last_delivered_id: Arc::new(AtomicU64::new(0)), nonrecursive_paths: Some(vec![root.clone()]), From 3301ce57707bc8473f0d95ab2b9908083e083a61 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:29:16 +0100 Subject: [PATCH 027/163] mason: allow deferred native FSEvents context destruction Native per-stream counters show an eventful stream can leave release_context pending when Drop returns, then release it exactly once as queue work drains. An eventless stream releases once as well. Keep independent-stream delivery assertions, but bound the deferred disconnection by five seconds and check both context release counters. No stream lifecycle or timing constants change. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit cc0752b0538a511fead97cf45541f44b3774d523) (cherry picked from commit 7346ef1f907bed24350db9e8deb7ea81c039a528) --- crates/aft/src/watcher_backend/fsevents.rs | 31 +++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/crates/aft/src/watcher_backend/fsevents.rs b/crates/aft/src/watcher_backend/fsevents.rs index 94642b9dc3..71f217e977 100644 --- a/crates/aft/src/watcher_backend/fsevents.rs +++ b/crates/aft/src/watcher_backend/fsevents.rs @@ -508,9 +508,10 @@ fn stream_action(queue: usize, stream: usize, start: bool) -> bool { start, started: false, }; - // Synchronous execution on the delivery queue fences callbacks: after a - // stop returns no callback can still be using the released stream context. - // Stream owners are backend/config workers, never native callbacks. + // Serialize lifecycle changes with native delivery. Invalidation stops + // callbacks, but FSEvents may retain the stream until queued work drains, + // releasing its context after this synchronous call returns. Stream owners + // are backend/config workers, never native callbacks. unsafe { dispatch_sync_f( queue as *mut _, @@ -1058,6 +1059,11 @@ mod tests { let start = Instant::now(); let deadline = start + Duration::from_secs(5); loop { + assert!( + Instant::now() < deadline, + "stream sender retained after five seconds; release_context calls={}", + releases.load(Ordering::SeqCst) + ); match rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) { Ok(_) => {} Err(mpsc::RecvTimeoutError::Disconnected) => break, @@ -1124,6 +1130,7 @@ mod tests { let mut streams = Vec::new(); let mut receivers = Vec::new(); let mut paths = Vec::new(); + let mut releases = Vec::new(); for root in &roots { let root = std::fs::canonicalize(root.path()).unwrap(); let (tx, rx) = mpsc::channel(); @@ -1136,6 +1143,7 @@ mod tests { ) .unwrap(); assert_eq!(stream.queue, delivery_queue().unwrap()); + releases.push(Arc::clone(&stream.context_release_calls)); paths.push(root.join("before-drop.rs")); streams.push(stream); receivers.push(rx); @@ -1145,22 +1153,15 @@ mod tests { wait_for_raw_path(rx, path); } drop(streams.remove(0)); - // Invalidation and release on the shared queue release the stream's - // callback context, so no sender or stream registration survives eviction. - while receivers[0].try_recv().is_ok() {} - assert!(matches!( - receivers[0].try_recv(), - Err(mpsc::TryRecvError::Disconnected) - )); + // FSEvents can retain a stream until queued work drains, so its context + // may be released after Drop returns. Bound that deferred release and + // verify that context release runs exactly once and drops its sender. + wait_for_context_release(&receivers[0], &releases[0]); let survivor = paths[1].with_file_name("after-drop.rs"); std::fs::write(&survivor, b"pub fn after() {}\n").unwrap(); wait_for_raw_path(&receivers[1], &survivor); drop(streams); - while receivers[1].try_recv().is_ok() {} - assert!(matches!( - receivers[1].try_recv(), - Err(mpsc::TryRecvError::Disconnected) - )); + wait_for_context_release(&receivers[1], &releases[1]); } fn symlink_replacement_fixture( From 43e871e3abf1679f1ad50dc8de7696939c27b332 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:21:00 +0100 Subject: [PATCH 028/163] mason: honor planned embedding pauses and quiet expected refusals Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 380b952905c26a1b0868d84da1993bfb5a78053f) (cherry picked from commit 9467eebc672c3831e0e1f9e0e0a982af8b37a50a) --- crates/aft/src/data_layer/meaning.rs | 6 + crates/aft/src/data_layer/runtime.rs | 253 ++++++++++++++++++ crates/aft/src/gh_shim_relay/mod.rs | 1 + crates/aft/src/gh_shim_speech/service.rs | 43 ++- .../aft/src/gh_shim_speech/service/tests.rs | 61 ++++- crates/aft/src/subc/consent.rs | 25 +- crates/aft/src/subc/consent_tests.rs | 35 +++ crates/aft/src/subc/diagnostics_findings.rs | 26 +- crates/aft/src/subc/health.rs | 25 ++ crates/aft/src/synapse_embed.rs | 54 +++- 10 files changed, 507 insertions(+), 22 deletions(-) diff --git a/crates/aft/src/data_layer/meaning.rs b/crates/aft/src/data_layer/meaning.rs index b7c7fe191c..5f6403839b 100644 --- a/crates/aft/src/data_layer/meaning.rs +++ b/crates/aft/src/data_layer/meaning.rs @@ -305,6 +305,12 @@ impl Meaning { lock(&self.config).clone().flatten() } + pub(super) fn synapse_lane(&self) -> Option { + self.config() + .filter(|config| config.backend == crate::config::SemanticBackend::Synapse) + .map(|config| config.model) + } + /// How meaning search stands for `view`, the checkout's current view /// (`None` before anything is published). pub(crate) fn state(&self, view: Option<&CheckoutView>) -> MeaningState { diff --git a/crates/aft/src/data_layer/runtime.rs b/crates/aft/src/data_layer/runtime.rs index 7030b5fb8b..53b0c7f657 100644 --- a/crates/aft/src/data_layer/runtime.rs +++ b/crates/aft/src/data_layer/runtime.rs @@ -333,6 +333,99 @@ struct BuildRetry { timer: Option, } +/// A scheduled lane pause has one clock and one wakeup, regardless of how +/// many checkouts are waiting for that embedding model. +#[derive(Default)] +struct FillPauses(std::collections::HashMap); + +struct FillPause { + pause: crate::synapse_embed::LanePause, + at: Instant, + waiters: std::collections::HashMap>, +} + +impl FillPauses { + /// Returns true when the lane needs its first wakeup thread. + fn record( + &mut self, + mut pause: crate::synapse_embed::LanePause, + now_ms: u64, + now: Instant, + ) -> bool { + if pause.paused_until_ms == 0 { + if self.0.get(&pause.lane_id).is_some_and(|entry| { + now < entry.at + && entry.pause.reason == pause.reason + && entry.pause.retry_after_ms == pause.retry_after_ms + }) { + return false; + } + pause.paused_until_ms = now_ms.saturating_add(pause.retry_after_ms.unwrap_or(0)); + } + let at = now + Duration::from_millis(pause.paused_until_ms.saturating_sub(now_ms)); + let first = !self.0.contains_key(&pause.lane_id); + let entry = self + .0 + .entry(pause.lane_id.clone()) + .or_insert_with(|| FillPause { + pause: pause.clone(), + at, + waiters: Default::default(), + }); + if first + || entry.pause.paused_until_ms != pause.paused_until_ms + || entry.pause.reason != pause.reason + { + crate::slog_info!( + "data layer meaning fill paused: lane={} until_ms={} reason={}", + pause.lane_id, + pause.paused_until_ms, + pause.reason + ); + entry.at = at; + entry.pause = pause; + } + first + } + + fn park(&mut self, lane: &str, root: &Path, work: Weak, now: Instant) -> bool { + let Some(entry) = self.0.get_mut(lane).filter(|entry| now < entry.at) else { + return false; + }; + crate::synapse_embed::share_lane_pause(root, &entry.pause); + entry.waiters.insert(root.to_path_buf(), work); + true + } +} + +fn fill_pauses() -> &'static parking_lot::Mutex { + static PAUSES: OnceLock> = OnceLock::new(); + PAUSES.get_or_init(Default::default) +} + +static FILL_PAUSE_CHANGED: parking_lot::Condvar = parking_lot::Condvar::new(); + +fn wake_paused_fills(lane: String) { + loop { + let waiters = { + let mut pauses = fill_pauses().lock(); + let Some(entry) = pauses.0.get(&lane) else { + return; + }; + let wait = entry.at.saturating_duration_since(Instant::now()); + if !wait.is_zero() { + FILL_PAUSE_CHANGED.wait_for(&mut pauses, wait); + continue; + } + pauses.0.remove(&lane).unwrap().waiters + }; + for work in waiters.into_values().filter_map(|work| work.upgrade()) { + work.start_fill(); + } + return; + } +} + /// A checkout and the state of its background work, shared by the work /// itself and the handle that starts it. struct Work { @@ -883,6 +976,9 @@ impl Work { if !self.meaning.enabled() || self.meaning.reads_only() { return; } + if self.park_for_lane_pause() { + return; + } { let retry = self .fill_retry @@ -902,6 +998,15 @@ impl Work { // Runs again when the root is bound and a search asks. return; } + // Another root may have learned of the pause while this job + // waited for a place in the process-wide fill pool. + if work.park_for_lane_pause() { + drop(flag); + // If the pause ended just as this job parked, its wakeup + // may have seen the running flag. Recheck after releasing it. + work.start_fill(); + return; + } let round = work.meaning.round(&work.checkout); drop(flag); work.notify_status_change(); @@ -928,6 +1033,34 @@ impl Work { } fn retry_fill_later(self: &Arc, error: &str) { + if let Some(pause) = crate::synapse_embed::LanePause::from_error(error) + .filter(|pause| pause.cause == crate::synapse_embed::PauseCause::LanePaused) + { + let lane = pause.lane_id.clone(); + let first = { + let mut pauses = fill_pauses().lock(); + let first = pauses.record( + pause, + crate::semantic_index::unix_millis_now(), + Instant::now(), + ); + // Also keep a waiter when an already-expired pause was returned: + // the lane's wakeup will start it immediately. + let entry = pauses.0.get_mut(&lane).unwrap(); + crate::synapse_embed::share_lane_pause(self.checkout.root(), &entry.pause); + entry + .waiters + .insert(self.checkout.root().to_path_buf(), Arc::downgrade(self)); + first + }; + FILL_PAUSE_CHANGED.notify_all(); + if first { + let _ = std::thread::Builder::new() + .name("aft-data-lane-resume".to_string()) + .spawn(move || wake_paused_fills(lane)); + } + return; + } let (wait, at) = { let mut retry = self .fill_retry @@ -961,6 +1094,17 @@ impl Work { } } + fn park_for_lane_pause(self: &Arc) -> bool { + self.meaning.synapse_lane().is_some_and(|lane| { + fill_pauses().lock().park( + &lane, + self.checkout.root(), + Arc::downgrade(self), + Instant::now(), + ) + }) + } + /// Tells the status listener, if any, that the checkout's status may /// have changed. fn notify_status_change(&self) { @@ -1777,6 +1921,115 @@ mod tests { use super::*; + fn planned_pause(until: u64) -> crate::synapse_embed::LanePause { + crate::synapse_embed::LanePause { + lane_id: "scheduled-embedding-lane".into(), + fingerprint: None, + retry_after_ms: Some(1_399_000), + paused_until_ms: until, + reason: "Neural Engine maintenance".into(), + cause: crate::synapse_embed::PauseCause::LanePaused, + safe_to_retry_same_request: true, + record_id: 0, + } + } + + #[test] + fn planned_pause_deadline_and_log_are_shared_across_fill_roots() { + let mut pauses = FillPauses::default(); + let now = Instant::now(); + let now_ms = crate::semantic_index::unix_millis_now(); + let pause = planned_pause(now_ms + 1_399_000); + let lane = pause.lane_id.clone(); + let first = tempfile::tempdir().unwrap(); + let second = tempfile::tempdir().unwrap(); + let (_, lines) = crate::logging::capture_log_lines(|| { + assert!(pauses.record(pause.clone(), now_ms, now)); + assert!(pauses.park(&lane, first.path(), Weak::new(), now)); + let mut duplicate = pause.clone(); + duplicate.record_id = 17; + duplicate.retry_after_ms = Some(1_390_000); + assert!(!pauses.record(duplicate, now_ms + 9_000, now + Duration::from_secs(9))); + assert!(pauses.park(&lane, second.path(), Weak::new(), now)); + }); + assert_eq!(pauses.0.len(), 1, "one lane timer, not one per root"); + let entry = &pauses.0[&lane]; + assert_eq!(entry.at, now + Duration::from_millis(1_399_000)); + assert_eq!(entry.waiters.len(), 2); + assert!(pauses.park( + &lane, + first.path(), + Weak::new(), + now + Duration::from_millis(1_398_999) + )); + assert!(!pauses.park( + &lane, + first.path(), + Weak::new(), + now + Duration::from_millis(1_399_000) + )); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!(lines[0].starts_with("INFO "), "{lines:?}"); + for value in [&lane, &pause.paused_until_ms.to_string(), &pause.reason] { + assert!(lines[0].contains(value), "{lines:?}"); + } + for root in [first.path(), second.path()] { + assert_eq!( + crate::synapse_embed::lane_pause_for_root(root), + Some(pause.clone()) + ); + } + let (_, lines) = crate::logging::capture_log_lines(|| { + assert!(!pauses.record(planned_pause(now_ms + 2_000_000), now_ms, now)); + }); + assert_eq!(lines.len(), 1, "a changed pause is reported once"); + assert_eq!(pauses.0[&lane].at, now + Duration::from_millis(2_000_000)); + } + + #[test] + fn planned_pause_without_absolute_end_uses_retry_hint() { + let mut pauses = FillPauses::default(); + let now = Instant::now(); + assert!(pauses.record(planned_pause(0), 1_000, now)); + let entry = &pauses.0["scheduled-embedding-lane"]; + assert_eq!(entry.pause.paused_until_ms, 1_400_000); + assert_eq!(entry.at, now + Duration::from_millis(1_399_000)); + assert!(!pauses.record(planned_pause(0), 10_000, now + Duration::from_secs(9))); + assert_eq!( + pauses.0["scheduled-embedding-lane"].at, + now + Duration::from_millis(1_399_000) + ); + } + + #[test] + fn planned_pause_fill_errors_bypass_per_root_failure_backoff() { + let first = PublicationFixture::new(Duration::from_secs(60), QUIET); + let second = PublicationFixture::new(Duration::from_secs(60), QUIET); + let mut pause = planned_pause(crate::semantic_index::unix_millis_now() + 1_399_000); + pause.lane_id = format!("fill-error-{}", first.handle.root().display()); + let error = format!("[lane_paused] {}", serde_json::to_string(&pause).unwrap()); + let (_, lines) = crate::logging::capture_log_lines(|| { + first.handle.work.retry_fill_later(&error); + second.handle.work.retry_fill_later(&error); + }); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!(lines[0].starts_with("INFO "), "{lines:?}"); + for fixture in [&first, &second] { + let retry = fixture.handle.work.fill_retry.lock().unwrap(); + assert_eq!(retry.in_a_row, 0); + assert!(retry.at.is_none()); + assert_eq!( + crate::synapse_embed::lane_pause_for_root(fixture.handle.root()), + Some(pause.clone()) + ); + } + let pauses = fill_pauses().lock(); + assert_eq!(pauses.0[&pause.lane_id].waiters.len(), 2); + assert!( + pauses.0[&pause.lane_id].at.duration_since(Instant::now()) > Duration::from_secs(1_390) + ); + } + struct PublicationFixture { handle: Arc, _root: tempfile::TempDir, diff --git a/crates/aft/src/gh_shim_relay/mod.rs b/crates/aft/src/gh_shim_relay/mod.rs index 008919f996..30c2fd60af 100644 --- a/crates/aft/src/gh_shim_relay/mod.rs +++ b/crates/aft/src/gh_shim_relay/mod.rs @@ -1213,6 +1213,7 @@ async fn post_captured_inner( &desired, &facade_reply(&reply), launch.agent()?, + &redeemed.session_id, )?; let expect = current_bindings.get(&repo).ok_or_else(|| { crate::gh_shim_speech::service::binding_refusal( diff --git a/crates/aft/src/gh_shim_speech/service.rs b/crates/aft/src/gh_shim_speech/service.rs index ef9219ca87..6c07900161 100644 --- a/crates/aft/src/gh_shim_speech/service.rs +++ b/crates/aft/src/gh_shim_speech/service.rs @@ -30,7 +30,7 @@ type EndNotice = Arc; pub(crate) fn binding_refusal(code: Code, reason: &str, detail: impl Into) -> Refusal { let detail = detail.into(); - log::warn!( + crate::slog_warn!( "gh speech binding verification refused: {}", serde_json::json!({"reason": reason, "detail": detail}) ); @@ -43,6 +43,35 @@ fn binding_unverifiable(detail: impl Into) -> Refusal { binding_refusal(Code::AuthorityUnavailable, "binding_unverifiable", detail) } +static UNCONFIGURED_BINDING_REFUSALS: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +pub(crate) fn unconfigured_binding_refusals_total() -> u64 { + UNCONFIGURED_BINDING_REFUSALS.load(std::sync::atomic::Ordering::Relaxed) +} + +fn unconfigured_binding_refusal(session: &str, agent: &str, detail: String) -> Refusal { + static LOGGED: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + UNCONFIGURED_BINDING_REFUSALS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + // Ordinary sessions need not bind any repository to the launching agent. + // Keep the refusal intact, but report that expected absence only once. + if LOGGED + .get_or_init(Mutex::default) + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert((session.to_owned(), agent.to_owned())) + { + crate::slog_info!( + "gh speech binding verification refused: {}", + serde_json::json!({"reason": "binding_unverifiable", "detail": detail, "session": session, "agent": agent}) + ); + } + let mut refusal = Refusal::new(Code::AuthorityUnavailable, "binding_unverifiable"); + refusal.detail = Some(detail); + refusal +} + #[cfg(test)] #[derive(Clone)] pub(crate) struct TestPartners { @@ -600,7 +629,7 @@ impl LaunchContext { ) }) .await?; - parse_bindings(&desired, &facade_reply(&reply), agent) + parse_bindings(&desired, &facade_reply(&reply), agent, &self.seed.session) }, ) .await @@ -637,6 +666,7 @@ pub(crate) fn parse_bindings( desired: &BTreeMap, reply: &Value, agent: &str, + session: &str, ) -> Result, Refusal> { let invalid = || binding_unverifiable("invalid_binding"); let reply = reply @@ -693,9 +723,12 @@ pub(crate) fn parse_bindings( .filter(|(_, owner)| owner.as_str() == agent) .map(|(repo, _)| repo.as_str()) .collect(); - return Err(binding_unverifiable(format!( - "no_binding_row: launch_agent={agent:?} repositories={expected:?}" - ))); + let detail = format!("no_binding_row: launch_agent={agent:?} repositories={expected:?}"); + return Err(if expected.is_empty() { + unconfigured_binding_refusal(session, agent, detail) + } else { + binding_unverifiable(detail) + }); } Ok(found) } diff --git a/crates/aft/src/gh_shim_speech/service/tests.rs b/crates/aft/src/gh_shim_speech/service/tests.rs index 201a8400e9..9b08a5b641 100644 --- a/crates/aft/src/gh_shim_speech/service/tests.rs +++ b/crates/aft/src/gh_shim_speech/service/tests.rs @@ -65,7 +65,7 @@ fn binding_reply_diagnostics_distinguish_missing_generation_missing_rows_and_inv "invalid_binding", ), ] { - let refusal = parse_bindings(&desired, &reply, "agent-subc").unwrap_err(); + let refusal = parse_bindings(&desired, &reply, "agent-subc", "session").unwrap_err(); assert_eq!(refusal.code, Code::AuthorityUnavailable); assert_eq!(refusal.reason, "binding_unverifiable"); assert!( @@ -83,6 +83,7 @@ fn no_binding_row_preserves_main_refusal_wording_and_structured_detail() { &desired, &serde_json::json!({"repo_binding_generation":36,"bindings":[]}), "agent-subc", + "session", ) .unwrap_err(); assert_eq!( @@ -97,6 +98,57 @@ fn no_binding_row_preserves_main_refusal_wording_and_structured_detail() { .starts_with("no_binding_row:")); } +#[test] +fn unconfigured_binding_refusal_is_info_once_per_session_and_agent() { + let desired = BTreeMap::from([("org/other".into(), "other-agent".into())]); + let reply = serde_json::json!({"repo_binding_generation":36,"bindings":[]}); + let before = unconfigured_binding_refusals_total(); + let (_, lines) = crate::logging::capture_log_lines(|| { + for (session, agent) in [ + ("unconfigured-session-a", "agent-a"), + ("unconfigured-session-a", "agent-a"), + ("unconfigured-session-b", "agent-a"), + ("unconfigured-session-a", "agent-b"), + ] { + let refusal = parse_bindings(&desired, &reply, agent, session).unwrap_err(); + assert_eq!(refusal.code, Code::AuthorityUnavailable); + assert_eq!(refusal.reason, "binding_unverifiable"); + assert_eq!( + refusal.detail.unwrap(), + format!("no_binding_row: launch_agent={agent:?} repositories=[]") + ); + } + }); + assert_eq!(lines.len(), 3, "{lines:?}"); + assert!( + lines.iter().all(|line| line.starts_with("INFO ")), + "{lines:?}" + ); + assert!( + unconfigured_binding_refusals_total() >= before + 4, + "count suppressed attempts too" + ); +} + +#[test] +fn configured_binding_absence_and_other_refusals_still_warn_each_time() { + let desired = BTreeMap::from([("org/repo".into(), "agent".into())]); + let reply = serde_json::json!({"repo_binding_generation":36,"bindings":[]}); + let (_, lines) = crate::logging::capture_log_lines(|| { + for _ in 0..2 { + parse_bindings(&desired, &reply, "agent", "configured-session").unwrap_err(); + } + binding_unverifiable("binding_read_timeout"); + }); + assert_eq!(lines.len(), 3, "{lines:?}"); + assert!( + lines.iter().all(|line| line.starts_with("WARN ")), + "{lines:?}" + ); + assert!(lines[0].contains("org/repo"), "{lines:?}"); + assert!(lines[2].contains("binding_read_timeout"), "{lines:?}"); +} + #[test] fn real_plexus_binding_replies_preserve_authority_and_refuse_missing_or_wrong_agents() { use crate::gh_shim_relay::bindings_contract::{vector, CASES}; @@ -107,7 +159,7 @@ fn real_plexus_binding_replies_preserve_authority_and_refuse_missing_or_wrong_ag ]); for name in CASES { let (_, reply) = vector(name); - let parsed = parse_bindings(&desired, &reply, "agent-x"); + let parsed = parse_bindings(&desired, &reply, "agent-x", "session"); match name { "agent" | "connection" => { let bindings = parsed.unwrap(); @@ -141,7 +193,8 @@ fn real_plexus_binding_replies_preserve_authority_and_refuse_missing_or_wrong_ag } } } - let bindings = parse_bindings(&desired, &vector("other-agent").1, "agent-y").unwrap(); + let bindings = + parse_bindings(&desired, &vector("other-agent").1, "agent-y", "session").unwrap(); assert_eq!( bindings.keys().map(String::as_str).collect::>(), ["o/a", "o/z"] @@ -158,7 +211,7 @@ fn binding_reply_diagnostics_name_both_agents_without_accepting_mismatches() { let reply = json!({"repo_binding_generation":36,"bindings":[{ "repository":"cortexkit/subconscious","agent_id":"agent-aft","app_handle_id":"aft-alfonso" }]}); - let refusal = parse_bindings(&desired, &reply, "agent-subc").unwrap_err(); + let refusal = parse_bindings(&desired, &reply, "agent-subc", "session").unwrap_err(); assert_eq!(refusal.code, Code::IdentityConflict); assert_eq!(refusal.reason, "binding_changed"); let detail = refusal.detail.unwrap(); diff --git a/crates/aft/src/subc/consent.rs b/crates/aft/src/subc/consent.rs index 164c8144e4..8a5d7b0522 100644 --- a/crates/aft/src/subc/consent.rs +++ b/crates/aft/src/subc/consent.rs @@ -2031,6 +2031,7 @@ impl DirectoryScopeProbe { struct DirectoryScopeProbes { probes: HashMap>, logged: HashMap<(PathBuf, &'static str), Instant>, + missing_logged: std::collections::HashSet, last_pruned: Option, } @@ -2057,6 +2058,9 @@ impl DirectoryScopeProbes { } fn note_fallback(&mut self, directory: &Path, reason: &'static str) -> bool { + if reason == "directory does not exist" { + return self.missing_logged.insert(directory.to_path_buf()); + } match self.logged.entry((directory.to_path_buf(), reason)) { std::collections::hash_map::Entry::Occupied(mut entry) => { if entry.get().elapsed() < DIRECTORY_SCOPE_MEMO_TTL { @@ -2128,16 +2132,29 @@ where .unwrap() .note_fallback(parent, reason); if should_log { - log::warn!( - "consent directory scope fell back to file-only for {}: {reason}", - parent.display() - ); + log_directory_scope_fallback(parent, reason); } grant.clone() } } } +fn log_directory_scope_fallback(parent: &Path, reason: &str) { + // Scratch directories may be removed after their command exits, before + // consent can offer a wider grant. The original file grant is still safe. + if reason == "directory does not exist" { + crate::slog_info!( + "consent directory scope fell back to file-only for {}: {reason}", + parent.display() + ); + } else { + crate::slog_warn!( + "consent directory scope fell back to file-only for {}: {reason}", + parent.display() + ); + } +} + async fn scope_before_db(grant: &AccessGrant) -> AccessGrant { let probing = grant.clone(); // A bounded helper still waits for its deadline. Keep that wait off the diff --git a/crates/aft/src/subc/consent_tests.rs b/crates/aft/src/subc/consent_tests.rs index a05bbfbbc8..aaf407e77d 100644 --- a/crates/aft/src/subc/consent_tests.rs +++ b/crates/aft/src/subc/consent_tests.rs @@ -4250,6 +4250,41 @@ fn consent_scope_unknown_alias_answer_is_not_memoized() { ); } +#[test] +fn consent_scope_vanished_directory_is_info_once_even_after_memo_pruning() { + let _lock = CONSENT_TEST_LOCK.blocking_lock(); + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().join("removed-scratch"); + let grant = AccessGrant { + path: parent.join("first.txt"), + kind: GrantKind::File, + operation: AccessOperation::Read, + }; + let (_, lines) = crate::logging::capture_log_lines(|| { + assert_eq!(directory_scope_with(&grant, |_, _| Some(false)), grant); + let mut state = DIRECTORY_SCOPE_PROBES.lock().unwrap(); + state.last_pruned = Some(Instant::now() - DIRECTORY_SCOPE_MEMO_TTL); + drop(state); + let sibling = AccessGrant { + path: parent.join("second.txt"), + ..grant.clone() + }; + assert_eq!(directory_scope_with(&sibling, |_, _| Some(false)), sibling); + }); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!(lines[0].starts_with("INFO "), "{lines:?}"); + assert!(lines[0].contains("directory does not exist"), "{lines:?}"); + let (_, lines) = crate::logging::capture_log_lines(|| { + log_directory_scope_fallback(&parent, "home identity unknown"); + log_directory_scope_fallback(&parent, "directory probe timed out"); + }); + assert_eq!(lines.len(), 2); + assert!( + lines.iter().all(|line| line.starts_with("WARN ")), + "{lines:?}" + ); +} + #[cfg(unix)] #[test] fn consent_scope_memo_expires_and_unsafe_answers_keep_each_file() { diff --git a/crates/aft/src/subc/diagnostics_findings.rs b/crates/aft/src/subc/diagnostics_findings.rs index 972a5e3916..d25951c9b7 100644 --- a/crates/aft/src/subc/diagnostics_findings.rs +++ b/crates/aft/src/subc/diagnostics_findings.rs @@ -292,9 +292,13 @@ pub(super) fn from_health( } } match metrics["embedding_backend"]["available"].as_bool() { + Some(false) if metrics["embedding_backend"]["state"] == "paused" && metrics["embedding_backend"]["pause_cause"] == "lane_paused" => findings.push(condition( + "semantic_backend", "paused", Severity::Info, 1, observed, now, + "The semantic embedding lane is intentionally paused.", vec![], + "Meaning fill will resume at health embedding_backend paused_until_ms; keyword search remains available.")), Some(false) => findings.push(condition("semantic_backend", "unavailable", Severity::Error, 1, metrics["embedding_backend"]["since_ms"].as_u64().unwrap_or(observed), now, - "The semantic embedding backend is unavailable or paused.", vec![], + "The semantic embedding backend is unavailable.", vec![], "Read health embedding_backend and semantic_index; inspect synapse availability and retry state.")), Some(true) => {}, None => findings.push(unknown("semantic_backend", "backend_state_unmeasured", now)), @@ -579,6 +583,26 @@ mod tests { assert!(findings.is_empty()); } + #[test] + fn diagnostics_findings_planned_lane_pause_is_information_not_failure() { + let mut metrics = healthy_metrics(); + metrics["embedding_backend"] = json!({"available": false, "state": "paused", "pause_cause": "lane_paused", "paused_until_ms": 2_000_000}); + let findings = snapshot(&metrics); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].check, "semantic_backend"); + assert_eq!(findings[0].key, "paused"); + assert_eq!(findings[0].severity, Severity::Info); + for backend in [ + json!({"available": false, "state": "backend_unavailable"}), + json!({"available": false, "state": "paused", "pause_cause": "transient"}), + ] { + metrics["embedding_backend"] = backend; + let findings = snapshot(&metrics); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].severity, Severity::Error); + } + } + #[test] fn diagnostics_findings_seeded_standing_conditions_and_redaction() { let mut metrics = healthy_metrics(); diff --git a/crates/aft/src/subc/health.rs b/crates/aft/src/subc/health.rs index 66870417e0..ed59a7f6b2 100644 --- a/crates/aft/src/subc/health.rs +++ b/crates/aft/src/subc/health.rs @@ -3491,6 +3491,7 @@ fn build_health_diagnostic_rollup_from_actors( let mut embedding_backend_available = true; let mut embedding_backend_paused_until_ms: Option = None; let mut embedding_backend_down = false; + let mut embedding_backend_only_planned_pauses = true; let mut embedding_backend_last_error = None; let mut embedding_backend_since_ms = None; let mut task_storage_roots = std::collections::HashSet::new(); @@ -3565,6 +3566,8 @@ fn build_health_diagnostic_rollup_from_actors( let backend = ctx.semantic_backend_health_snapshot(); if !backend.available { embedding_backend_available = false; + embedding_backend_only_planned_pauses &= + backend.pause_cause == Some(crate::synapse_embed::PauseCause::LanePaused); if let Some(until) = backend.paused_until_ms { embedding_backend_paused_until_ms = Some(embedding_backend_paused_until_ms.map_or(until, |old| old.max(until))); @@ -3887,6 +3890,7 @@ fn build_health_diagnostic_rollup_from_actors( "available": embedding_backend_available, "state": if embedding_backend_available { "available" } else if !embedding_backend_down && embedding_backend_paused_until_ms.is_some() { "paused" } else { "backend_unavailable" }, "paused_until_ms": embedding_backend_paused_until_ms, + "pause_cause": if !embedding_backend_available && !embedding_backend_down && embedding_backend_only_planned_pauses { Some("lane_paused") } else { None }, "last_error": embedding_backend_last_error, "since_ms": embedding_backend_since_ms, }, @@ -4233,6 +4237,10 @@ fn assemble_health_report( "durable_log".to_string(), crate::logging::durable_log_health(), ); + metrics.insert( + "gh_speech".to_string(), + json!({"unconfigured_binding_refusals_total": crate::gh_shim_speech::service::unconfigured_binding_refusals_total()}), + ); metrics.insert( "runtime".to_string(), json!({ @@ -8271,6 +8279,23 @@ mod tests { ); } + #[test] + fn health_report_exposes_unconfigured_binding_refusal_counter() { + let executor = Executor::new(); + let metrics = DispatchPathMetrics::new(); + let app = App::default_shared(); + let report = build_health_report( + &HealthRollupCache::new(), + &executor, + &HashMap::new(), + &metrics, + &app, + ); + assert!(report.metrics.expect("health metrics")["gh_speech"] + ["unconfigured_binding_refusals_total"] + .is_u64()); + } + #[test] fn health_report_lists_stopped_routes() { let executor = Executor::new(); diff --git a/crates/aft/src/synapse_embed.rs b/crates/aft/src/synapse_embed.rs index 99fcf79a2d..6783a8214a 100644 --- a/crates/aft/src/synapse_embed.rs +++ b/crates/aft/src/synapse_embed.rs @@ -61,6 +61,7 @@ pub struct LanePause { pub lane_id: String, pub fingerprint: Option, pub retry_after_ms: Option, + #[serde(default)] pub paused_until_ms: u64, pub reason: String, #[serde(default)] @@ -297,6 +298,15 @@ pub(crate) fn lane_pause_for_root(root: &Path) -> Option { Some(pause) } +/// A fill that waits on another root's lane pause reports the same planned +/// downtime in health, even though it made no request of its own. +pub(crate) fn share_lane_pause(root: &Path, pause: &LanePause) { + lane_pauses() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(root.to_path_buf(), pause.clone()); +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum SynapseEmbeddingError { MissingConnectionFile, @@ -1549,20 +1559,28 @@ fn decode_result_envelope_for_request( }); } } - let paused_until_ms = details["paused_until_ms"].as_u64().ok_or_else(|| { - SynapseEmbeddingError::InvalidEnvelope( - "lane_paused omitted paused_until_ms".into(), - ) - })?; + let retry_after_ms = error["retry_after_ms"] + .as_u64() + .filter(|ms| (1_000..=MAX_LANE_PAUSE_MS).contains(ms)); + let paused_until_ms = details["paused_until_ms"] + .as_u64() + .or_else(|| { + retry_after_ms.map(|delay| { + crate::semantic_index::unix_millis_now().saturating_add(delay) + }) + }) + .ok_or_else(|| { + SynapseEmbeddingError::InvalidEnvelope( + "lane_paused omitted paused_until_ms".into(), + ) + })?; let reason = details["reason"].as_str().ok_or_else(|| { SynapseEmbeddingError::InvalidEnvelope("lane_paused omitted reason".into()) })?; Err(SynapseEmbeddingError::LanePaused(LanePause { lane_id: lane_id.to_owned(), fingerprint: fingerprint.and_then(Value::as_str).map(str::to_owned), - retry_after_ms: error["retry_after_ms"] - .as_u64() - .filter(|ms| (1_000..=MAX_LANE_PAUSE_MS).contains(ms)), + retry_after_ms, paused_until_ms, reason: reason.to_owned(), cause: PauseCause::LanePaused, @@ -2109,6 +2127,26 @@ mod tests { )); } + #[test] + fn lane_paused_retry_hint_supplies_missing_absolute_end() { + use super::pause_test_support::*; + let mut reply = pause_reply(json!(1_399_000)); + reply["error"]["details"] + .as_object_mut() + .unwrap() + .remove("paused_until_ms"); + let before = crate::semantic_index::unix_millis_now(); + let error = decode_result_envelope_for_request( + serde_json::to_vec(&reply).unwrap(), + Some(&json!({"model": MODEL, "required_fingerprint": "pinned"})), + ) + .unwrap_err(); + let after = crate::semantic_index::unix_millis_now(); + let pause = LanePause::from_error(&error.to_string()).unwrap(); + assert!((before + 1_399_000..=after + 1_399_000).contains(&pause.paused_until_ms)); + assert_eq!(pause.cause, PauseCause::LanePaused); + } + #[test] fn lane_paused_fake_synapse_preserves_breaker_and_reports_health() { use super::pause_test_support::*; From 803eb7ab50f56db9397c6a4b93afcd60d855f972 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:39:45 +0100 Subject: [PATCH 029/163] mason: keep planned pauses outside failure-only findings Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 2e5145b2f09d42b31d0dbb79aaeb33d478e28bd3) (cherry picked from commit 9a9954aafb46ed8cb8b2522bbc672b8ba5d05e32) --- crates/aft/src/subc/diagnostics_findings.rs | 14 +++--- crates/aft/src/subc/health.rs | 48 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/crates/aft/src/subc/diagnostics_findings.rs b/crates/aft/src/subc/diagnostics_findings.rs index d25951c9b7..1cdfb34f1a 100644 --- a/crates/aft/src/subc/diagnostics_findings.rs +++ b/crates/aft/src/subc/diagnostics_findings.rs @@ -292,10 +292,8 @@ pub(super) fn from_health( } } match metrics["embedding_backend"]["available"].as_bool() { - Some(false) if metrics["embedding_backend"]["state"] == "paused" && metrics["embedding_backend"]["pause_cause"] == "lane_paused" => findings.push(condition( - "semantic_backend", "paused", Severity::Info, 1, observed, now, - "The semantic embedding lane is intentionally paused.", vec![], - "Meaning fill will resume at health embedding_backend paused_until_ms; keyword search remains available.")), + // Planned downtime is information in health, not a warn/error finding. + Some(false) if metrics["embedding_backend"]["state"] == "paused" && metrics["embedding_backend"]["pause_cause"] == "lane_paused" => {}, Some(false) => findings.push(condition("semantic_backend", "unavailable", Severity::Error, 1, metrics["embedding_backend"]["since_ms"].as_u64().unwrap_or(observed), now, "The semantic embedding backend is unavailable.", vec![], @@ -588,10 +586,10 @@ mod tests { let mut metrics = healthy_metrics(); metrics["embedding_backend"] = json!({"available": false, "state": "paused", "pause_cause": "lane_paused", "paused_until_ms": 2_000_000}); let findings = snapshot(&metrics); - assert_eq!(findings.len(), 1); - assert_eq!(findings[0].check, "semantic_backend"); - assert_eq!(findings[0].key, "paused"); - assert_eq!(findings[0].severity, Severity::Info); + assert!( + findings.is_empty(), + "planned pauses are not failures: {findings:?}" + ); for backend in [ json!({"available": false, "state": "backend_unavailable"}), json!({"available": false, "state": "paused", "pause_cause": "transient"}), diff --git a/crates/aft/src/subc/health.rs b/crates/aft/src/subc/health.rs index ed59a7f6b2..036c847860 100644 --- a/crates/aft/src/subc/health.rs +++ b/crates/aft/src/subc/health.rs @@ -8747,6 +8747,54 @@ mod tests { assert_eq!(semantic["query_embed_p50_ms"], 200); } + #[test] + fn semantic_backend_planned_pause_is_information_but_does_not_hide_outages() { + let executor = Executor::new(); + let (dir, root) = test_root("semantic-planned-pause-health"); + let ctx = Arc::new(AppContext::new( + Box::new(crate::parser::TreeSitterProvider::new()), + crate::config::Config { + storage_dir: Some(dir.path().join("storage")), + project_root: Some(root.as_path().into()), + ..crate::config::Config::default() + }, + )); + let until = crate::semantic_index::unix_millis_now() + 1_399_000; + crate::synapse_embed::share_lane_pause( + root.as_path(), + &crate::synapse_embed::LanePause { + lane_id: "health-planned-lane".into(), + fingerprint: None, + retry_after_ms: Some(1_399_000), + paused_until_ms: until, + reason: "Neural Engine maintenance".into(), + cause: crate::synapse_embed::PauseCause::LanePaused, + safe_to_retry_same_request: true, + record_id: 0, + }, + ); + assert!(executor.register_actor(root, ctx)); + let app = App::default_shared(); + let cache = HealthRollupCache::new(); + let report = refresh_until_root_count(&cache, &executor, &app, 1) + .metrics + .unwrap(); + assert_eq!(report["embedding_backend"]["available"], false); + assert_eq!(report["embedding_backend"]["state"], "paused"); + assert_eq!(report["embedding_backend"]["pause_cause"], "lane_paused"); + assert_eq!(report["embedding_backend"]["paused_until_ms"], until); + + let (_outage_dir, outage_root) = test_root("semantic-outage-with-pause-health"); + let outage = test_ctx(); + outage.trip_semantic_refresh_circuit(1, "connection refused"); + assert!(executor.register_actor(outage_root, outage)); + let report = refresh_until_root_count(&cache, &executor, &app, 2) + .metrics + .unwrap(); + assert_eq!(report["embedding_backend"]["state"], "backend_unavailable"); + assert!(report["embedding_backend"]["pause_cause"].is_null()); + } + #[test] fn semantic_backend_outage_is_distinct_per_root_and_aggregated_once() { let executor = Executor::new(); From 471ecb25288bc5d8dc1a4e259441ee64d3723e0c Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:53:53 +0100 Subject: [PATCH 030/163] mason: initialize canonical root in pause health fixture Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 1a34a6ded1caab8774af79bb60474823b711dca8) (cherry picked from commit be14228dc74dcbd96591de4c919c5750da04fa99) --- crates/aft/src/subc/health.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/aft/src/subc/health.rs b/crates/aft/src/subc/health.rs index 036c847860..8bfa21e56d 100644 --- a/crates/aft/src/subc/health.rs +++ b/crates/aft/src/subc/health.rs @@ -8759,6 +8759,7 @@ mod tests { ..crate::config::Config::default() }, )); + ctx.set_canonical_cache_root(root.as_path().to_path_buf()); let until = crate::semantic_index::unix_millis_now() + 1_399_000; crate::synapse_embed::share_lane_pause( root.as_path(), From 2866d9915f8b82674cb764917fec9d93bfd6a375 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:12:30 +0100 Subject: [PATCH 031/163] mason: bound expected-refusal log deduplication history Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 160635ada7ed182be6cdb0648978962e73fe1908) (cherry picked from commit 0f6e1abff4113286809185b760fe9e9f0c07b901) --- crates/aft/src/gh_shim_speech/service.rs | 48 ++++++++++++++----- .../aft/src/gh_shim_speech/service/tests.rs | 23 +++++++++ crates/aft/src/list_surfaces.rs | 12 +++++ crates/aft/src/subc/consent.rs | 10 ++++ crates/aft/src/subc/consent_tests.rs | 20 ++++++++ 5 files changed, 101 insertions(+), 12 deletions(-) diff --git a/crates/aft/src/gh_shim_speech/service.rs b/crates/aft/src/gh_shim_speech/service.rs index 6c07900161..04b7c11d33 100644 --- a/crates/aft/src/gh_shim_speech/service.rs +++ b/crates/aft/src/gh_shim_speech/service.rs @@ -50,28 +50,52 @@ pub(crate) fn unconfigured_binding_refusals_total() -> u64 { UNCONFIGURED_BINDING_REFUSALS.load(std::sync::atomic::Ordering::Relaxed) } +const UNCONFIGURED_BINDING_LOG_CAP: usize = 4_096; + fn unconfigured_binding_refusal(session: &str, agent: &str, detail: String) -> Refusal { static LOGGED: std::sync::OnceLock>> = std::sync::OnceLock::new(); UNCONFIGURED_BINDING_REFUSALS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); // Ordinary sessions need not bind any repository to the launching agent. - // Keep the refusal intact, but report that expected absence only once. - if LOGGED - .get_or_init(Mutex::default) - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert((session.to_owned(), agent.to_owned())) - { - crate::slog_info!( - "gh speech binding verification refused: {}", - serde_json::json!({"reason": "binding_unverifiable", "detail": detail, "session": session, "agent": agent}) - ); - } + // Keep the refusal intact and suppress repeats while the log key is retained. + log_unconfigured_binding_refusal( + &mut LOGGED + .get_or_init(Mutex::default) + .lock() + .unwrap_or_else(|error| error.into_inner()), + session, + agent, + &detail, + UNCONFIGURED_BINDING_LOG_CAP, + ); let mut refusal = Refusal::new(Code::AuthorityUnavailable, "binding_unverifiable"); refusal.detail = Some(detail); refusal } +fn log_unconfigured_binding_refusal( + logged: &mut HashSet<(String, String)>, + session: &str, + agent: &str, + detail: &str, + cap: usize, +) { + let key = (session.to_owned(), agent.to_owned()); + if logged.contains(&key) { + return; + } + // Bound history to 4,096 session/agent keys in the long-lived daemon. + // A new key clears a full set; an older session may log once more after that. + if logged.len() >= cap { + logged.clear(); + } + logged.insert(key); + crate::slog_info!( + "gh speech binding verification refused: {}", + serde_json::json!({"reason": "binding_unverifiable", "detail": detail, "session": session, "agent": agent}) + ); +} + #[cfg(test)] #[derive(Clone)] pub(crate) struct TestPartners { diff --git a/crates/aft/src/gh_shim_speech/service/tests.rs b/crates/aft/src/gh_shim_speech/service/tests.rs index 9b08a5b641..be2925e9ed 100644 --- a/crates/aft/src/gh_shim_speech/service/tests.rs +++ b/crates/aft/src/gh_shim_speech/service/tests.rs @@ -128,6 +128,29 @@ fn unconfigured_binding_refusal_is_info_once_per_session_and_agent() { unconfigured_binding_refusals_total() >= before + 4, "count suppressed attempts too" ); + + let mut logged = HashSet::new(); + let (_, lines) = crate::logging::capture_log_lines(|| { + for (session, expected_keys) in [ + ("session-a", 1), + ("session-a", 1), + ("session-b", 2), + ("session-b", 2), + ("session-c", 1), + ("session-a", 2), + ] { + log_unconfigured_binding_refusal(&mut logged, session, "agent", "no_binding_row", 2); + assert_eq!(logged.len(), expected_keys); + } + }); + assert_eq!(lines.len(), 4, "a cleared session may log again: {lines:?}"); + assert!( + lines.iter().all(|line| line.starts_with("INFO ")), + "{lines:?}" + ); + assert!(lines[0].contains("session-a"), "{lines:?}"); + assert!(lines[2].contains("session-c"), "{lines:?}"); + assert!(lines[3].contains("session-a"), "{lines:?}"); } #[test] diff --git a/crates/aft/src/list_surfaces.rs b/crates/aft/src/list_surfaces.rs index f447e7fa72..7a38756f03 100644 --- a/crates/aft/src/list_surfaces.rs +++ b/crates/aft/src/list_surfaces.rs @@ -364,6 +364,18 @@ pub static EXCLUSIONS: &[ExclusionEntry] = &[ location_or_primitive: ".take(1_000)", reason: "Internal permission-error scan stops after 1,000 matching lines to select one explanatory note; it never truncates the command output or returns a result list.", }, + ExclusionEntry { + file: "gh_shim_speech/service.rs", + enclosing_item: "log_unconfigured_binding_refusal", + location_or_primitive: "UNCONFIGURED_BINDING_LOG_CAP, cap, clear", + reason: "Internal INFO-log deduplication history is cleared at capacity; older sessions may log again, but refusals and agent-visible result lists are unchanged.", + }, + ExclusionEntry { + file: "subc/consent.rs", + enclosing_item: "DirectoryScopeProbes::note_fallback", + location_or_primitive: "DIRECTORY_SCOPE_MISSING_LOG_CAP, clear", + reason: "Internal vanished-directory INFO-log history is cleared at capacity; older scopes may log again, without changing the original file grant or any agent-visible result list.", + }, ExclusionEntry { file: "gh_shim_speech/start_view/sweep.rs", enclosing_item: "sweep_entries", diff --git a/crates/aft/src/subc/consent.rs b/crates/aft/src/subc/consent.rs index 8a5d7b0522..8ca2953ac1 100644 --- a/crates/aft/src/subc/consent.rs +++ b/crates/aft/src/subc/consent.rs @@ -2027,6 +2027,8 @@ impl DirectoryScopeProbe { } } +const DIRECTORY_SCOPE_MISSING_LOG_CAP: usize = 4_096; + #[derive(Default)] struct DirectoryScopeProbes { probes: HashMap>, @@ -2059,6 +2061,14 @@ impl DirectoryScopeProbes { fn note_fallback(&mut self, directory: &Path, reason: &'static str) -> bool { if reason == "directory does not exist" { + if self.missing_logged.contains(directory) { + return false; + } + // Bound vanished-scope log history in the long-lived daemon. + // A new scope clears the full set; older scopes may then log again. + if self.missing_logged.len() >= DIRECTORY_SCOPE_MISSING_LOG_CAP { + self.missing_logged.clear(); + } return self.missing_logged.insert(directory.to_path_buf()); } match self.logged.entry((directory.to_path_buf(), reason)) { diff --git a/crates/aft/src/subc/consent_tests.rs b/crates/aft/src/subc/consent_tests.rs index aaf407e77d..5f6c9cda37 100644 --- a/crates/aft/src/subc/consent_tests.rs +++ b/crates/aft/src/subc/consent_tests.rs @@ -4283,6 +4283,26 @@ fn consent_scope_vanished_directory_is_info_once_even_after_memo_pruning() { lines.iter().all(|line| line.starts_with("WARN ")), "{lines:?}" ); + + let mut probes = DirectoryScopeProbes::default(); + for index in 0..DIRECTORY_SCOPE_MISSING_LOG_CAP { + assert!(probes.note_fallback(&parent.join(index.to_string()), "directory does not exist")); + assert_eq!(probes.missing_logged.len(), index + 1); + } + assert!(!probes.note_fallback(&parent.join("0"), "directory does not exist")); + assert_eq!(probes.missing_logged.len(), DIRECTORY_SCOPE_MISSING_LOG_CAP); + assert!(probes.note_fallback(&parent.join("next"), "directory does not exist")); + assert_eq!( + probes.missing_logged.len(), + 1, + "a new scope clears the full set" + ); + assert!(probes.note_fallback(&parent.join("0"), "directory does not exist")); + assert_eq!( + probes.missing_logged.len(), + 2, + "an older scope may log again" + ); } #[cfg(unix)] From 75d64230e5e57dace6530531a3516996782bf46d Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:58:12 +0100 Subject: [PATCH 032/163] mason: tolerate CI stalls in slow embedding recovery test Scale this test's delay and deadline floor together by ten, preserving the timeout ladder while giving a singleton 2.5 seconds of scheduling margin. Leave the shared 300 ms fixture unchanged so the never-answer and other timing tests retain their budgets. Keep all three 64-row calls and strengthen the recovery assertion: the final call must start at 64 and succeed as one request, not silently downshift or rely on a late server write. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 84ff220561c31c8020ff73541632b6d48541c791) (cherry picked from commit b04d764226daf9e71f2e0b830460876a3f76758d) --- crates/aft/src/semantic_index.rs | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/crates/aft/src/semantic_index.rs b/crates/aft/src/semantic_index.rs index bb33ece3d1..c112afa96a 100644 --- a/crates/aft/src/semantic_index.rs +++ b/crates/aft/src/semantic_index.rs @@ -15468,17 +15468,36 @@ public class Greeter { #[test] fn slow_backend_converges_without_being_marked_down() { - // The reporter's shape (2 s/item against a 25 s floor) scaled so the - // test exercises real HTTP deadlines in seconds, not minutes: a 64-item - // batch cannot fit the initial deadline, a 4-item batch can. - let server = ProgrammableEmbeddingServer::start(Duration::from_millis(50)); - let config = programmable_http_config(&server); + // A 3 s floor leaves a singleton 2.5 s of scheduling margin on loaded + // CI runners. Scale the per-item delay with it: a 64-item batch still + // takes 32 s against its initial 12 s deadline, while 4 items fit in + // the 3 s floor. The per-item latency EMA (exponential moving average) + // must raise the deadline to recover to 64. + // Keep the shared config's short deadlines for the never-answer test. + let server = ProgrammableEmbeddingServer::start(Duration::from_millis(500)); + let mut config = programmable_http_config(&server); + config.timeout_ms = 3_000; let mut model = SemanticEmbeddingModel::from_config(&config).unwrap(); - for _ in 0..3 { + for _ in 0..2 { assert_eq!(model.embed(embedding_inputs(64)).unwrap().len(), 64); } + assert_eq!(model.adaptive_build_batch_size, config.max_batch_size); + assert!( + server.request_sizes().starts_with(&[64, 32, 16, 8, 4]), + "the initial deadline must force smaller batches: requests={:?}", + server.request_sizes(), + ); + // Start at the recovered size and require the client to receive all + // rows; a timed-out server handler can still finish writing a response. + let requests_before_recovery = server.request_sizes().len(); + assert_eq!(model.embed(embedding_inputs(64)).unwrap().len(), 64); + assert_eq!( + &server.request_sizes()[requests_before_recovery..], + &[64], + "the recovered 64-item request must complete without another downshift", + ); assert_eq!(model.adaptive_build_batch_size, config.max_batch_size); assert!( server.completed_sizes().contains(&64), From 82aa0b52a2317a874abad3baa841be16e20ba56d Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:09:35 +0100 Subject: [PATCH 033/163] mason: keep slow embedding test within a one-minute budget Use the requested fivefold scale (250 ms per item and a 1,500 ms floor) rather than tenfold so serial Windows shards pay about 60 seconds, not 120. The singleton work/floor ratio remains 1/6 with 1.25 seconds of margin; initial 64-item deadline/work remains 3/8; four-item work/floor remains 2/3; recovered EMA deadline/work remains about 2. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit c71e1dfe10b4fbab97510fb3bc1be0b5a5dee0cf) (cherry picked from commit 7a92ba58e45e47daade9ead607190a0d6f3fcbb6) --- crates/aft/src/semantic_index.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/aft/src/semantic_index.rs b/crates/aft/src/semantic_index.rs index c112afa96a..7d1953bb6d 100644 --- a/crates/aft/src/semantic_index.rs +++ b/crates/aft/src/semantic_index.rs @@ -15468,15 +15468,15 @@ public class Greeter { #[test] fn slow_backend_converges_without_being_marked_down() { - // A 3 s floor leaves a singleton 2.5 s of scheduling margin on loaded + // A 1.5 s floor leaves a singleton 1.25 s of scheduling margin on loaded // CI runners. Scale the per-item delay with it: a 64-item batch still - // takes 32 s against its initial 12 s deadline, while 4 items fit in - // the 3 s floor. The per-item latency EMA (exponential moving average) + // takes 16 s against its initial 6 s deadline, while 4 items fit in + // the 1.5 s floor. The per-item latency EMA (exponential moving average) // must raise the deadline to recover to 64. // Keep the shared config's short deadlines for the never-answer test. - let server = ProgrammableEmbeddingServer::start(Duration::from_millis(500)); + let server = ProgrammableEmbeddingServer::start(Duration::from_millis(250)); let mut config = programmable_http_config(&server); - config.timeout_ms = 3_000; + config.timeout_ms = 1_500; let mut model = SemanticEmbeddingModel::from_config(&config).unwrap(); for _ in 0..2 { From ae239dab36e346f017a3e2dc566fa638b78bfe30 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:24:30 +0100 Subject: [PATCH 034/163] mason: add bounded attested secret-use transport and contract fixture Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 6b2fd606b4fb437d19006097f00ba1c0cb3d8e65) (cherry picked from commit bb9583f992f4725a531bc32be024c4fc4e4d076f) --- .../release-notes/next/live-secret-vault.md | 1 + crates/aft/src/secret/mod.rs | 5 +- crates/aft/src/secret/transport.rs | 154 ++++++++ crates/aft/src/secret/transport_fake.rs | 363 ++++++++++++++++++ crates/aft/src/secret/transport_tests.rs | 272 +++++++++++++ crates/aft/src/secret/wire.rs | 100 ++++- crates/aft/src/secret/wire_tests.rs | 47 +++ .../tests/fixtures/secret/secret-use-v1.jsonl | 6 + 8 files changed, 938 insertions(+), 10 deletions(-) create mode 100644 .alfonso/release-notes/next/live-secret-vault.md create mode 100644 crates/aft/src/secret/transport.rs create mode 100644 crates/aft/src/secret/transport_fake.rs create mode 100644 crates/aft/src/secret/transport_tests.rs create mode 100644 crates/aft/tests/fixtures/secret/secret-use-v1.jsonl diff --git a/.alfonso/release-notes/next/live-secret-vault.md b/.alfonso/release-notes/next/live-secret-vault.md new file mode 100644 index 0000000000..a615251f6a --- /dev/null +++ b/.alfonso/release-notes/next/live-secret-vault.md @@ -0,0 +1 @@ +Added the bounded, daemon-authenticated transport and contract fixtures for live vault secret use. Shell secret placeholders remain unavailable until the request-addressed vault client is connected. diff --git a/crates/aft/src/secret/mod.rs b/crates/aft/src/secret/mod.rs index c3cccb58cd..fdf0d41db4 100644 --- a/crates/aft/src/secret/mod.rs +++ b/crates/aft/src/secret/mod.rs @@ -18,6 +18,7 @@ pub mod consent; pub mod eligibility; pub mod outcome; pub mod scope; +pub mod transport; pub mod vault; pub mod wire; @@ -38,8 +39,8 @@ pub use vault::{ }; pub use wire::{ parse_answer, parse_reply, ApiKeyTarget, ApprovalRequest, Coverage, LoginTarget, - ReleasedApiKey, ReleasedLogin, ReplyError, SecretUseAnswer, SecretUseReply, SecretUseRequest, - Target, VaultError, VaultTransportError, + ReleasedApiKey, ReleasedLogin, ReplyError, SecretUseAnswer, SecretUseRefusal, SecretUseReply, + SecretUseRequest, Target, VaultError, VaultTransportError, }; #[cfg(any(test, feature = "secret-test-fake"))] diff --git a/crates/aft/src/secret/transport.rs b/crates/aft/src/secret/transport.rs new file mode 100644 index 0000000000..cc4b74c3b5 --- /dev/null +++ b/crates/aft/src/secret/transport.rs @@ -0,0 +1,154 @@ +//! Bounded `secret.use` calls over the daemon's attested consumer routes. + +use std::path::PathBuf; +use std::time::Duration; + +use serde_json::{json, Value}; +use subc_client_rs::{CallError, CallOptions, ConsumerIdentity, ConsumerOptions}; +use subc_protocol::scope::{ScopeSelector, ScopeStamp}; +use subc_protocol::{BindIdentity, RouteTarget}; +use zeroize::Zeroizing; + +use super::vault::SecretUseTransport; + +const UNAVAILABLE: &str = "secret.use transport unavailable"; + +/// A transport for a future request-addressed vault client. Construct it at call +/// time: a standalone process has no module identity and must not open as direct. +/// No connection or vault availability is cached across calls. +#[derive(Clone)] +pub struct SubcSecretUseTransport { + connection_file: PathBuf, + identity: BindIdentity, + timeout: Duration, +} + +impl SubcSecretUseTransport { + /// Use AFT's captured launch-nonce identity, never an agent-supplied principal. + /// The deadline includes connection, scoped open, request, and cleanup. + pub fn new( + connection_file: PathBuf, + identity: BindIdentity, + timeout: Duration, + ) -> Option { + aft_identity()?; + Some(Self { + connection_file, + identity, + timeout, + }) + } + + async fn send(&self, params: Value, principal: ConsumerIdentity) -> Result { + let target = ¶ms["target"]; + if target["scheme"] != "https" + || target + .get("port") + .is_some_and(|port| port.as_u64() != Some(443)) + { + return Err(UNAVAILABLE.to_owned()); + } + let stamp: ScopeStamp = + serde_json::from_value(params.get("stamp").cloned().unwrap_or_default()) + .map_err(|_| UNAVAILABLE.to_owned())?; + let scope = ScopeSelector::new(stamp.owner, stamp.scope_ref, Some(stamp.scope_epoch)); + let body = serde_json::to_vec(&json!({"method": "secret.use", "params": params})) + .map_err(|_| UNAVAILABLE.to_owned())?; + let consumer = crate::fleet_status::connect_subc_consumer( + &self.connection_file, + ConsumerOptions::default(), + ) + .await + .map_err(|_| UNAVAILABLE.to_owned())?; + // Each call owns its consumer, so closing a route cannot interrupt a + // concurrent use of an SDK-cached handle. Dropping the call's runtime on + // timeout also drops the connection and all of its routes. + let result = async { + let route = consumer + .open_route_scoped( + RouteTarget::ManagementSurface { + module_id: "claustrum".into(), + }, + self.identity.clone(), + scope, + CallOptions { + consumer_identity: Some(principal), + timeout: self.timeout, + route_retry_deadline: self.timeout, + ..CallOptions::default() + }, + ) + .await + .map_err(|_| UNAVAILABLE.to_owned())?; + let reply = consumer + .request( + &route, + body, + CallOptions { + timeout: self.timeout, + ..CallOptions::default() + }, + ) + .await; + let result = match reply { + Ok(bytes) => { + let bytes = Zeroizing::new(bytes); + serde_json::from_slice(&bytes).map_err(|_| UNAVAILABLE.to_owned()) + } + // Preserve the distinction between a module Error frame and a + // result refusal. Do not include SDK errors or reply bytes in + // transport diagnostics: either could carry released material. + Err(CallError::Module(error)) => Ok(json!({ + "code": error.code, + "message": error.message, + })), + Err(_) => Err(UNAVAILABLE.to_owned()), + }; + let _ = consumer + .close_handle(&route, subc_client_rs::CloseRouteOptions::default()) + .await; + result + } + .await; + let _ = consumer.close().await; + result + } + + fn dispatch(&self, params: Value, principal: ConsumerIdentity) -> Result { + let this = self.clone(); + // The seam is synchronous and is also used outside Tokio. A dedicated + // runtime avoids entering a nested runtime on the caller's executor thread. + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| UNAVAILABLE.to_owned())?; + runtime.block_on(async { + tokio::time::timeout(this.timeout, this.send(params, principal)) + .await + .map_err(|_| UNAVAILABLE.to_owned())? + }) + }) + .join() + .map_err(|_| UNAVAILABLE.to_owned())? + } +} + +fn aft_identity() -> Option { + crate::launch_nonce::consumer_identity().filter(|identity| identity.module_id == "aft") +} + +impl SecretUseTransport for SubcSecretUseTransport { + fn secret_use(&self, params: Value) -> Result { + let principal = aft_identity().ok_or_else(|| UNAVAILABLE.to_owned())?; + self.dispatch(params, principal) + } +} + +#[cfg(test)] +#[path = "transport_tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "transport_fake.rs"] +pub(crate) mod fake; diff --git a/crates/aft/src/secret/transport_fake.rs b/crates/aft/src/secret/transport_fake.rs new file mode 100644 index 0000000000..4a7991287e --- /dev/null +++ b/crates/aft/src/secret/transport_fake.rs @@ -0,0 +1,363 @@ +//! A single-record claustrum fixture over authenticated subc route plumbing. +//! Approval is an explicit test action representing a verified phone signature; +//! neither retrying a request nor opening another route grants access. + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use base64::Engine; +use serde::Deserialize; +use serde_json::{json, Value}; +use subc_protocol::scope::{ParentState, ScopeKind, ScopeStamp}; +use subc_protocol::{Flags, Frame, FrameType, Priority}; +use subc_transport::connection_file::{self, ConnectionInfo, Endpoint, SCHEMA_VERSION}; + +use super::super::wire::{Coverage, Target}; + +const GOLDENS: &str = include_str!("../../tests/fixtures/secret/secret-use-v1.jsonl"); +pub(crate) const NONCE: &str = "fake-aft-launch-nonce"; + +pub(crate) fn golden(index: usize) -> Value { + serde_json::from_str(GOLDENS.lines().nth(index).unwrap()).unwrap() +} + +pub(crate) fn refusal(code: &str, class: &str) -> Value { + json!({"result":{"error":{"code":code,"class":class}}}) +} + +fn invalid() -> Value { + json!({"code":"invalid_params","message":"invalid params"}) +} + +#[derive(Default)] +pub(crate) struct State { + pub opens: Vec, + pub calls: usize, + pub hold_open: bool, + pub hold_reply: bool, + pub disconnect: bool, + pub store_error: bool, + pub setup_incomplete: bool, + pub other_pending_in_scope: usize, + pub other_pending_total: usize, + approved: HashSet, + pub(super) pending: HashMap, +} + +impl State { + /// Stand in for the vault verifying the phone's signature over these bytes. + pub fn approve(&mut self, statement_b64url: &str) { + let keys: Vec<_> = self + .pending + .iter() + .filter(|(_, reply)| reply["statement_b64url"] == statement_b64url) + .map(|(key, _)| key.clone()) + .collect(); + assert!(!keys.is_empty()); + self.approved.extend(keys); + } + + fn serve(&mut self, principal: &str, params: Value) -> Value { + let not_found = || refusal("not_found", "permanent"); + if !matches!(principal, "aft" | "cerebellum") { + return not_found(); + } + let Ok(params) = serde_json::from_value::(params) else { + return invalid(); + }; + if params.name.is_empty() + || !params.name.as_bytes()[0].is_ascii_alphanumeric() + || !params + .name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)) + { + return invalid(); + } + if params.name.len() > 64 { + return refusal("limit_exceeded", "context_overflow"); + } + if let Err(error) = valid_target(¶ms.target) { + return match error { + secret_rules::scope::ScopeRefusal::LimitExceeded => { + refusal("limit_exceeded", "context_overflow") + } + _ => invalid(), + }; + } + let stamp: ScopeStamp = match serde_json::from_value(params.stamp) { + Ok(stamp) => stamp, + Err(error) if error.to_string().starts_with("missing field ") => return not_found(), + Err(_) => return invalid(), + }; + if stamp.scope_epoch > (1 << 53) - 1 + || stamp + .parent + .as_ref() + .is_some_and(|p| p.scope_epoch > (1 << 53) - 1) + { + return invalid(); + } + if stamp.attributes.flow_id.is_some() { + return not_found(); + } + let (owner, kind, reference, epoch) = match stamp.kind { + ScopeKind::Head => (stamp.owner, "head", stamp.scope_ref, stamp.scope_epoch), + ScopeKind::Worker => (stamp.owner, "worker", stamp.scope_ref, stamp.scope_epoch), + ScopeKind::Ephemeral => { + let Some(parent) = stamp.parent.filter(|parent| { + stamp.parent_state == Some(ParentState::Linked) + && parent.scope_epoch == stamp.scope_epoch + && parent.owner + == (subc_protocol::Principal::Reserved { + module_id: "prefrontal-core".into(), + }) + }) else { + return not_found(); + }; + (parent.owner, "head", parent.scope_ref, parent.scope_epoch) + } + }; + let coverage = params.coverage; + if coverage == Coverage::AgentAndWorkers && kind != "head" { + return invalid(); + } + let matches = match ¶ms.target { + Target::ApiKey(target) => { + principal == "aft" + && params.name == "key" + && target.scheme == "https" + && target.port.unwrap_or(443) == 443 + && target.host == "x.test" + && (target.path == "/repos/a" || target.path.starts_with("/repos/a/")) + } + Target::Login(_) => false, + }; + if !matches { + return not_found(); + } + if self.store_error { + return refusal("store_error", "transient"); + } + let owner = match owner { + subc_protocol::Principal::Reserved { module_id } => format!("reserved:{module_id}"), + subc_protocol::Principal::Direct => "direct".into(), + subc_protocol::Principal::Unverified => "unverified".into(), + }; + let key = format!( + "{}:{owner}:{kind}:{reference}:{}", + params.name, + coverage.as_str() + ); + if self.approved.contains(&key) { + return json!({"result":golden(3)}); + } + if self.setup_incomplete { + return refusal("secret_setup_incomplete", "permanent"); + } + if let Some(pending) = self.pending.get(&key) { + return json!({"result":pending}); + } + if self.other_pending_in_scope >= 8 || self.other_pending_total + self.pending.len() >= 64 { + return refusal("pending_limit", "transient"); + } + let mut pending = golden(5); + pending["request_id"] = json!(format!("01HF7YAT{:018}", self.pending.len() + 1)); + let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; + let mut statement: Value = serde_json::from_slice( + &engine + .decode(pending["statement_b64url"].as_str().unwrap()) + .unwrap(), + ) + .unwrap(); + statement["request_id"] = pending["request_id"].clone(); + statement["scope"] = json!({"epoch":epoch,"kind":kind,"owner":owner,"ref":reference}); + statement["coverage"] = json!(coverage.as_str()); + if coverage == Coverage::AgentAndWorkers { + statement["durations"] = json!(["permanent"]); + } + statement["agent_id"] = json!(stamp.attributes.agent_id); + pending["statement_b64url"] = json!(engine.encode(serde_json::to_vec(&statement).unwrap())); + self.pending.insert(key, pending.clone()); + json!({"result":pending}) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Params { + name: String, + #[serde(default = "default_coverage")] + coverage: Coverage, + stamp: Value, + target: Target, +} + +fn default_coverage() -> Coverage { + Coverage::Scope +} + +fn valid_target(target: &Target) -> Result<(), secret_rules::scope::ScopeRefusal> { + use secret_rules::scope::{ScopeRefusal, SecretScope}; + match target { + Target::ApiKey(target) => { + if target.scheme.is_empty() + || !target + .scheme + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b"+-.".contains(&b)) + { + return Err(ScopeRefusal::InvalidParams); + } + SecretScope::parse_input("agent_api_key", &json!({"host":target.host}))?; + if !valid_path(&target.path) { + return Err(ScopeRefusal::InvalidParams); + } + Ok(()) + } + Target::Login(target) => secret_rules::scope::canonical_origin(&target.origin).map(|_| ()), + } +} + +fn valid_path(path: &str) -> bool { + if !path.starts_with('/') { + return false; + } + let mut bytes = path.bytes(); + while let Some(byte) = bytes.next() { + if byte == b'%' { + let Some(a) = bytes.next().and_then(|b| (b as char).to_digit(16)) else { + return false; + }; + let Some(b) = bytes.next().and_then(|b| (b as char).to_digit(16)) else { + return false; + }; + let decoded = (a * 16 + b) as u8; + if !(0x21..0x7f).contains(&decoded) || b"/.%\\;".contains(&decoded) { + return false; + } + } else if !(byte.is_ascii_alphanumeric() || b"-._~!$&'()*+,;=:@/".contains(&byte)) { + return false; + } + } + let segments: Vec<_> = path[1..].split('/').collect(); + segments.iter().enumerate().all(|(i, segment)| { + (!segment.is_empty() || i == segments.len() - 1) + && !matches!(segment.split(';').next(), Some("." | "..")) + }) +} + +pub(crate) struct FakeClaustrum { + _directory: tempfile::TempDir, + pub path: PathBuf, + pub state: Arc>, + stop: Option>, + server: Option>, +} + +impl FakeClaustrum { + pub fn start() -> Self { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("connection.json"); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let key = vec![0x42; subc_transport::KEY_LEN]; + let daemon_id = [0x24; subc_transport::DAEMON_ID_LEN]; + connection_file::write_atomic( + &path, + &ConnectionInfo { + schema: SCHEMA_VERSION, + wire_version: Some(subc_protocol::PROTOCOL_VERSION), + endpoints: vec![Endpoint { + host: "127.0.0.1".into(), + port: listener.local_addr().unwrap().port(), + }], + key: key.clone(), + daemon_id, + pid: std::process::id(), + daemon_ver: "0.39.0".into(), + }, + ) + .unwrap(); + let state = Arc::new(Mutex::new(State::default())); + let served = Arc::clone(&state); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(async move { + let listener = tokio::net::TcpListener::from_std(listener).unwrap(); + let mut tasks = tokio::task::JoinSet::new(); + tokio::pin!(stopped); + loop { + tokio::select! { + _ = &mut stopped => break, + accepted = listener.accept() => { + let (mut stream, _) = accepted.unwrap(); + let state = Arc::clone(&served); + let key = key.clone(); + tasks.spawn(async move { + subc_transport::authenticate_server(&mut stream, &key, &daemon_id, "0.39.0", Duration::from_secs(2)).await.unwrap(); + let mut principal = String::new(); + while let Ok(Some(request)) = subc_transport::read_frame(&mut stream).await { + if request.header.ty == FrameType::Ping { + let pong = Frame::build(FrameType::Pong, flags(), 0, 0, request.header.corr, Vec::new()).unwrap(); + if subc_transport::write_frame(&mut stream, &pong).await.is_err() { break; } + continue; + } + if request.header.ty != FrameType::Request { continue; } + let value: Value = serde_json::from_slice(&request.body).unwrap(); + let (reply, error) = if value["op"] == "route.open" { + assert_eq!(value["target"], json!({"kind":"management_surface","module_id":"claustrum"})); + assert!(value.get("scope").is_some()); + let hold = { + let mut state = state.lock().unwrap(); + state.opens.push(value.clone()); + state.hold_open + }; + if hold { continue; } + if value["consumer_identity"]["launch_nonce"] == NONCE { + principal = value["consumer_identity"]["module_id"].as_str().unwrap().into(); + } + (json!({"op":"route.open","route_channel":7,"route_epoch":1}), false) + } else { + assert_eq!(value["method"], "secret.use"); + let mut state = state.lock().unwrap(); + state.calls += 1; + if state.disconnect { break; } + if state.hold_reply { continue; } + let reply = state.serve(&principal, value["params"].clone()); + let error = reply.get("code").is_some(); + (reply, error) + }; + let frame = Frame::build(if error { FrameType::Error } else { FrameType::Response }, flags(), request.header.channel, request.header.epoch, request.header.corr, serde_json::to_vec(&reply).unwrap()).unwrap(); + if subc_transport::write_frame(&mut stream, &frame).await.is_err() { break; } + } + }); + } + Some(result) = tasks.join_next(), if !tasks.is_empty() => result.unwrap(), + } + } + tasks.abort_all(); + }); + }); + Self { + _directory: directory, + path, + state, + stop: Some(stop), + server: Some(server), + } + } +} + +impl Drop for FakeClaustrum { + fn drop(&mut self) { + let _ = self.stop.take().unwrap().send(()); + self.server.take().unwrap().join().unwrap(); + } +} + +fn flags() -> Flags { + Flags::new(false, Priority::Interactive, false) +} diff --git a/crates/aft/src/secret/transport_tests.rs b/crates/aft/src/secret/transport_tests.rs new file mode 100644 index 0000000000..cd74df720f --- /dev/null +++ b/crates/aft/src/secret/transport_tests.rs @@ -0,0 +1,272 @@ +use super::fake::{golden, refusal, FakeClaustrum, NONCE}; +use super::*; +use crate::secret::{parse_answer, SecretUseAnswer, SecretUseRefusal, SecretUseReply}; + +fn principal(module_id: &str) -> ConsumerIdentity { + ConsumerIdentity { + module_id: module_id.into(), + launch_nonce: NONCE.into(), + } +} + +fn transport(daemon: &FakeClaustrum) -> SubcSecretUseTransport { + SubcSecretUseTransport { + connection_file: daemon.path.clone(), + identity: BindIdentity::new("aft-secret-use", "secret-call"), + timeout: Duration::from_secs(2), + } +} + +fn call(transport: &SubcSecretUseTransport, params: Value) -> Value { + transport.dispatch(params, principal("aft")).unwrap() +} + +#[test] +fn secret_use_route_preserves_the_stamp_and_releases_only_after_phone_approval() { + let daemon = FakeClaustrum::start(); + let transport = transport(&daemon); + let pending = call(&transport, golden(0)); + assert_eq!(pending, json!({"result":golden(5)})); + assert_eq!( + call(&transport, golden(1)), + pending, + "defaults recover the same statement bytes" + ); + daemon + .state + .lock() + .unwrap() + .approve(pending["result"]["statement_b64url"].as_str().unwrap()); + let released = call(&transport, golden(0)); + assert_eq!(released, json!({"result":golden(3)})); + let SecretUseAnswer::Reply(SecretUseReply::ReleasedApiKey(key)) = + parse_answer(&released).unwrap() + else { + panic!("the approved API key must be released"); + }; + assert_eq!(key.material(), b"sk-example"); + assert!(!format!("{key:?}").contains("sk-example")); + assert!(!format!("{key:?}").contains("c2stZXhhbXBsZQ")); + let state = daemon.state.lock().unwrap(); + assert_eq!(state.calls, 3); + assert_eq!(state.opens.len(), 3); + for open in &state.opens { + assert_eq!( + open["consumer_identity"], + json!({"module_id":"aft","launch_nonce":NONCE}) + ); + let stamp: ScopeStamp = serde_json::from_value(golden(0)["stamp"].clone()).unwrap(); + assert_eq!( + open["scope"], + json!(ScopeSelector::new( + stamp.owner, + stamp.scope_ref, + Some(stamp.scope_epoch) + )) + ); + } +} + +#[test] +fn secret_use_route_keeps_module_errors_distinct_from_uniform_not_found() { + let daemon = FakeClaustrum::start(); + let transport = transport(&daemon); + let mut extra_name = golden(0); + extra_name["display_name"] = json!("not trusted"); + let rejected = call(&transport, extra_name); + assert_eq!( + parse_answer(&rejected).unwrap().refusal(), + Some(SecretUseRefusal::InvalidParams) + ); + + let mut unknown = golden(0); + unknown["name"] = json!("unknown"); + let not_found = call(&transport, unknown); + assert_eq!(not_found, refusal("not_found", "permanent")); + let mut mismatch = golden(0); + mismatch["target"]["host"] = json!("other.test"); + assert_eq!(call(&transport, mismatch), not_found); + let mut mismatch = golden(0); + mismatch["target"]["path"] = json!("/repos/another"); + assert_eq!(call(&transport, mismatch), not_found); + assert!(daemon.state.lock().unwrap().pending.is_empty()); +} + +#[test] +fn secret_use_route_maps_all_sender_refusals_without_releasing_material() { + use SecretUseRefusal::*; + let daemon = FakeClaustrum::start(); + let transport = transport(&daemon); + let mut oversized = golden(0); + oversized["name"] = json!("a".repeat(65)); + let limit = call(&transport, oversized); + assert_eq!(limit, refusal("limit_exceeded", "context_overflow")); + assert_eq!(parse_answer(&limit).unwrap().refusal(), Some(LimitExceeded)); + for expected in [SetupIncomplete, PendingLimit, StoreError] { + { + let mut state = daemon.state.lock().unwrap(); + state.setup_incomplete = expected == SetupIncomplete; + state.other_pending_in_scope = if expected == PendingLimit { 8 } else { 0 }; + state.store_error = expected == StoreError; + } + let answer = parse_answer(&call(&transport, golden(0))).unwrap(); + assert_eq!(answer.refusal(), Some(expected)); + } +} + +#[test] +fn secret_use_deadline_covers_scoped_open_reply_and_disconnect_without_secret_errors() { + for phase in ["open", "reply", "disconnect", "no-daemon"] { + let daemon = FakeClaustrum::start(); + let mut transport = transport(&daemon); + transport.timeout = Duration::from_millis(150); + { + let mut state = daemon.state.lock().unwrap(); + state.hold_open = phase == "open"; + state.hold_reply = phase == "reply"; + state.disconnect = phase == "disconnect"; + } + if phase == "no-daemon" { + transport.connection_file = daemon.path.with_extension("absent"); + } + let started = std::time::Instant::now(); + let error = transport.dispatch(golden(0), principal("aft")).unwrap_err(); + assert_eq!(error, UNAVAILABLE, "{phase}"); + assert!( + started.elapsed() < Duration::from_secs(2), + "{phase} exceeded the call deadline" + ); + assert!(!error.contains("sk-example")); + assert!(!error.contains("c2stZXhhbXBsZQ")); + } +} + +#[test] +fn secret_use_sender_never_sends_a_non_https_or_non_443_target() { + let daemon = FakeClaustrum::start(); + let transport = transport(&daemon); + for (scheme, port) in [("http", 443), ("https", 8443)] { + let mut params = golden(0); + params["target"]["scheme"] = json!(scheme); + params["target"]["port"] = json!(port); + assert_eq!( + transport.dispatch(params, principal("aft")).unwrap_err(), + UNAVAILABLE + ); + } + assert_eq!(daemon.state.lock().unwrap().opens.len(), 0); +} + +#[test] +fn secret_use_standalone_cannot_construct_a_module_transport_and_keeps_unavailable_vault() { + let _nonce = crate::launch_nonce::install_for_tests(None); + assert!(SubcSecretUseTransport::new( + PathBuf::from("absent-connection.json"), + BindIdentity::new("standalone", "standalone"), + Duration::from_millis(150), + ) + .is_none()); + let stamp: ScopeStamp = serde_json::from_value(golden(0)["stamp"].clone()).unwrap(); + let vault = crate::bash_background::secrets::vault(); + assert!(matches!( + vault.resolve(&crate::secret::SecretScope::from_stamp(&stamp), "key"), + crate::secret::Resolution::Unavailable + )); +} + +// Negative contract inputs deliberately bypass the sender's typed construction. +// They still travel through the SDK's authenticated connection, scoped open, and +// request path, rather than calling the fake endpoint's handler directly. +fn raw_call(daemon: &FakeClaustrum, identity: Option, params: Value) -> Value { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let consumer = crate::fleet_status::connect_subc_consumer( + &daemon.path, + ConsumerOptions::default(), + ) + .await + .unwrap(); + let stamp: ScopeStamp = serde_json::from_value(golden(0)["stamp"].clone()).unwrap(); + let route = consumer + .open_route_scoped( + RouteTarget::ManagementSurface { + module_id: "claustrum".into(), + }, + BindIdentity::new("secret-contract", "negative"), + ScopeSelector::new(stamp.owner, stamp.scope_ref, Some(stamp.scope_epoch)), + CallOptions { + consumer_identity: identity, + timeout: Duration::from_secs(1), + ..CallOptions::default() + }, + ) + .await + .unwrap(); + let reply = consumer + .request( + &route, + serde_json::to_vec(&json!({"method":"secret.use","params":params})).unwrap(), + CallOptions { + timeout: Duration::from_secs(1), + ..CallOptions::default() + }, + ) + .await; + let reply = match reply { + Ok(bytes) => serde_json::from_slice(&bytes).unwrap(), + Err(CallError::Module(error)) => json!({"code":error.code,"message":error.message}), + Err(error) => panic!("fake route failed: {error}"), + }; + let _ = consumer.close().await; + reply + }) +} + +#[test] +fn fake_claustrum_orders_caller_grammar_scope_and_target_checks() { + let daemon = FakeClaustrum::start(); + let missing = refusal("not_found", "permanent"); + assert_eq!( + raw_call(&daemon, Some(principal("other")), json!(null)), + missing + ); + assert_eq!( + raw_call(&daemon, None, golden(0)), + missing, + "direct callers are not AFT" + ); + let mut unstamped = golden(0); + unstamped["stamp"] + .as_object_mut() + .unwrap() + .remove("owner_authorized"); + assert_eq!( + raw_call(&daemon, Some(principal("aft")), unstamped), + missing + ); + for (field, value) in [("scheme", json!("http")), ("port", json!(8443))] { + let mut mismatched = golden(0); + mismatched["target"][field] = value; + assert_eq!( + raw_call(&daemon, Some(principal("aft")), mismatched), + missing + ); + } + for target in [ + json!({"scheme":"HTTP","host":"x.test","path":"/repos/a"}), + json!({"scheme":"https","host":"x.test","path":"/repos/a/%2e"}), + json!({"scheme":"https","host":"x.test","path":"/repos/a/..;x"}), + json!({"scheme":"https","host":"x.test","path":"/repos/a?secret"}), + ] { + let mut bad = golden(0); + bad["target"] = target; + let answer = raw_call(&daemon, Some(principal("aft")), bad); + assert_eq!( + parse_answer(&answer).unwrap().refusal(), + Some(SecretUseRefusal::InvalidParams) + ); + } +} diff --git a/crates/aft/src/secret/wire.rs b/crates/aft/src/secret/wire.rs index 3106822293..2e966010b3 100644 --- a/crates/aft/src/secret/wire.rs +++ b/crates/aft/src/secret/wire.rs @@ -30,7 +30,7 @@ pub const COVERAGE_SCOPE: &str = "scope"; pub const COVERAGE_AGENT_AND_WORKERS: &str = "agent_and_workers"; /// Which approval coverage a request asks for. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum Coverage { /// One secret use for the asking scope (the default when omitted). @@ -53,28 +53,35 @@ impl Coverage { /// /// Senders strip the query and fragment before calling, so a `?` or `#` in /// `path` is a grammar fault on the vault's side; AFT refuses to send one. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ApiKeyTarget { /// `https`, the only scheme the vault accepts. pub scheme: String, /// Lowercase ASCII host. pub host: String, /// The port; omitted on the wire when it is the default 443. - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "present_option" + )] pub port: Option, /// The request path, starting with `/` and carrying no query or fragment. pub path: String, } /// The target a login was asked for: `{origin}`, canonical `https://host`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct LoginTarget { /// The canonical origin, with port 443 omitted. pub origin: String, } /// The `target` member of a `secret.use` request. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(untagged)] pub enum Target { /// An API key for one host, port and path. ApiKey(ApiKeyTarget), @@ -96,12 +103,17 @@ impl Serialize for Target { /// The `stamp` is the daemon's `ScopeStamp` value, relayed byte for byte. AFT /// never rewrites it, because the vault trusts the sender to pass it through /// unchanged and takes the asking scope from nowhere else. -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct SecretUseRequest { /// The secret name (`[A-Za-z0-9][A-Za-z0-9._-]*`, 1–64 bytes). pub name: String, /// The coverage; `scope` when omitted. - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "present_option" + )] pub coverage: Option, /// The asking call's daemon stamp, unchanged. pub stamp: ScopeStamp, @@ -109,6 +121,12 @@ pub struct SecretUseRequest { pub target: Target, } +fn present_option<'de, D: serde::Deserializer<'de>, T: Deserialize<'de>>( + deserializer: D, +) -> Result, D::Error> { + T::deserialize(deserializer).map(Some) +} + impl SecretUseRequest { /// Build a request for the scope `stamp` names. /// @@ -227,7 +245,7 @@ pub enum SecretUseReply { } /// The vault's pending approval (§15 "Signed statement and envelope"). -#[derive(Clone, PartialEq, Eq)] +#[derive(Clone, PartialEq, Eq, Serialize)] pub struct ApprovalRequest { /// The vault's identifier for the pending request, a 26-character ULID. pub request_id: String, @@ -284,6 +302,72 @@ pub enum SecretUseAnswer { Rejected(VaultTransportError), } +/// Refusals in the sender contract's ordered checks. A `not_found` never +/// distinguishes caller admission, scope mapping, record, kind, or target. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SecretUseRefusal { + NotFound, + InvalidParams, + LimitExceeded, + SetupIncomplete, + PendingLimit, + StoreError, +} + +impl SecretUseAnswer { + /// Classify known sender refusals without discarding the original wire + /// code/class. Unknown future refusals remain refusals, not success. + pub fn refusal(&self) -> Option { + use SecretUseRefusal::*; + match self { + Self::Rejected(error) if error.code == "invalid_params" => Some(InvalidParams), + Self::Refused(error) => match error.code.as_str() { + "not_found" => Some(NotFound), + "limit_exceeded" => Some(LimitExceeded), + "secret_setup_incomplete" => Some(SetupIncomplete), + "pending_limit" => Some(PendingLimit), + "store_error" => Some(StoreError), + _ => None, + }, + _ => None, + } + } +} + +impl Serialize for SecretUseReply { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::SerializeMap; + let mut map = serializer.serialize_map(None)?; + match self { + Self::ReleasedApiKey(key) => { + let encoded = zeroize::Zeroizing::new(b64url().encode(key.material())); + map.serialize_entry("status", "released")?; + map.serialize_entry("kind", "agent_api_key")?; + map.serialize_entry("key_b64url", encoded.as_str())?; + map.serialize_entry("location", &key.location)?; + } + Self::ReleasedLogin(login) => { + let password = std::str::from_utf8(login.password()) + .map_err(|_| serde::ser::Error::custom("login password is not UTF-8"))?; + map.serialize_entry("status", "released")?; + map.serialize_entry("kind", "site_login")?; + map.serialize_entry("username", &login.username)?; + map.serialize_entry("password", password)?; + if let Some(totp) = &login.totp { + map.serialize_entry("totp", totp)?; + } + } + Self::ApprovalRequired(request) => { + map.serialize_entry("status", "approval_required")?; + map.serialize_entry("request_id", &request.request_id)?; + map.serialize_entry("statement_b64url", &request.statement_b64url)?; + map.serialize_entry("expires_at_ms", &request.expires_at_ms)?; + } + } + map.end() + } +} + /// The base64url engine the contract uses: strict and unpadded. fn b64url() -> base64::engine::general_purpose::GeneralPurpose { base64::engine::general_purpose::URL_SAFE_NO_PAD diff --git a/crates/aft/src/secret/wire_tests.rs b/crates/aft/src/secret/wire_tests.rs index dc4915237d..aa996e6f4a 100644 --- a/crates/aft/src/secret/wire_tests.rs +++ b/crates/aft/src/secret/wire_tests.rs @@ -18,6 +18,53 @@ const VECTORS: &str = include_str!("../../tests/fixtures/secret/secret-approval- /// The sha256 the campaign pinned for the vector file. const VECTORS_SHA256: &str = "efd4ba60641586e143bce81087e4f9f5aaf8a99ffede74190303da7d4495cc27"; +// The six agent-sender request/result examples in claustrum's wire-contract v1 +// §15, in document order. Keep their literal JSON bytes, including statement bytes. +pub(super) const USE_GOLDENS: &str = + include_str!("../../tests/fixtures/secret/secret-use-v1.jsonl"); + +#[test] +fn every_sender_golden_round_trips_through_the_wire_types() { + let examples: Vec<_> = USE_GOLDENS.lines().collect(); + assert_eq!(examples.len(), 6); + for (index, example) in examples.into_iter().enumerate() { + let value: Value = serde_json::from_str(example).unwrap(); + let encoded = if index < 3 { + let request: SecretUseRequest = serde_json::from_value(value).unwrap(); + serde_json::to_string(&request).unwrap() + } else { + let reply = parse_reply(&value).unwrap(); + serde_json::to_string(&reply).unwrap() + }; + assert_eq!(encoded, example, "sender golden {index}"); + } +} + +#[test] +fn sender_refusals_preserve_the_contract_code_class_and_error_frame() { + use SecretUseRefusal::*; + for (code, class, refusal) in [ + ("not_found", "permanent", NotFound), + ("limit_exceeded", "context_overflow", LimitExceeded), + ("secret_setup_incomplete", "permanent", SetupIncomplete), + ("pending_limit", "transient", PendingLimit), + ("store_error", "transient", StoreError), + ] { + let answer = + parse_answer(&json!({"result":{"error":{"code":code,"class":class}}})).unwrap(); + assert_eq!(answer.refusal(), Some(refusal)); + let SecretUseAnswer::Refused(error) = answer else { + panic!("result refusal") + }; + assert_eq!(error.code, code); + assert_eq!(error.class.as_deref(), Some(class)); + } + let answer = + parse_answer(&json!({"code":"invalid_params","message":"invalid params"})).unwrap(); + assert_eq!(answer.refusal(), Some(InvalidParams)); + assert!(matches!(answer, SecretUseAnswer::Rejected(_))); +} + fn vectors() -> Value { serde_json::from_str(VECTORS).expect("the pinned vector file is JSON") } diff --git a/crates/aft/tests/fixtures/secret/secret-use-v1.jsonl b/crates/aft/tests/fixtures/secret/secret-use-v1.jsonl new file mode 100644 index 0000000000..b3fa0ed74b --- /dev/null +++ b/crates/aft/tests/fixtures/secret/secret-use-v1.jsonl @@ -0,0 +1,6 @@ +{"name":"key","coverage":"scope","stamp":{"owner":{"kind":"reserved","module_id":"prefrontal-core"},"ref":"H","scope_epoch":7,"kind":"head","attributes":{"agent_id":"agent-A"},"owner_authorized":true},"target":{"scheme":"https","host":"x.test","port":443,"path":"/repos/a/issues/1"}} +{"name":"key","stamp":{"owner":{"kind":"reserved","module_id":"prefrontal-core"},"ref":"H","scope_epoch":7,"kind":"head","attributes":{"agent_id":"agent-A"},"owner_authorized":true},"target":{"scheme":"https","host":"x.test","path":"/repos/a/issues/1"}} +{"name":"login","stamp":{"owner":{"kind":"reserved","module_id":"prefrontal-core"},"ref":"W","scope_epoch":7,"kind":"worker","parent":{"owner":{"kind":"reserved","module_id":"prefrontal-core"},"ref":"H","scope_epoch":7},"parent_state":"linked","attributes":{"agent_id":"agent-A"},"owner_authorized":true},"target":{"origin":"https://x.test"}} +{"status":"released","kind":"agent_api_key","key_b64url":"c2stZXhhbXBsZQ","location":"header:X-Api-Key"} +{"status":"released","kind":"site_login","username":"alice","password":"correct horse","totp":"123456"} +{"status":"approval_required","request_id":"01HF7YAT000000000000000001","statement_b64url":"eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0","expires_at_ms":1700000600000} From 307daad9152b3af447225f08f1813934b6cf81b5 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:34:50 +0100 Subject: [PATCH 035/163] mason: make secret-use fixture handle cancelled authentication and grant coverage Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 79320ad7ae0acbf0cc549792f293d179b0086192) (cherry picked from commit 4c7d07462a2a573569e2e59081becbf8ccf1b9f6) --- crates/aft/src/secret/transport_fake.rs | 71 ++++++++++++++++++------ crates/aft/src/secret/transport_tests.rs | 26 +++++++++ crates/aft/src/secret/wire_tests.rs | 5 +- 3 files changed, 83 insertions(+), 19 deletions(-) diff --git a/crates/aft/src/secret/transport_fake.rs b/crates/aft/src/secret/transport_fake.rs index 4a7991287e..79638dc060 100644 --- a/crates/aft/src/secret/transport_fake.rs +++ b/crates/aft/src/secret/transport_fake.rs @@ -19,6 +19,8 @@ use super::super::wire::{Coverage, Target}; const GOLDENS: &str = include_str!("../../tests/fixtures/secret/secret-use-v1.jsonl"); pub(crate) const NONCE: &str = "fake-aft-launch-nonce"; +type PendingKey = (String, String, String, String, String); + pub(crate) fn golden(index: usize) -> Value { serde_json::from_str(GOLDENS.lines().nth(index).unwrap()).unwrap() } @@ -42,8 +44,8 @@ pub(crate) struct State { pub setup_incomplete: bool, pub other_pending_in_scope: usize, pub other_pending_total: usize, - approved: HashSet, - pub(super) pending: HashMap, + approved: HashSet, + pub(super) pending: HashMap, } impl State { @@ -103,6 +105,14 @@ impl State { if stamp.attributes.flow_id.is_some() { return not_found(); } + let worker_head = stamp + .parent + .as_ref() + .filter(|parent| { + stamp.parent_state == Some(ParentState::Linked) + && parent.scope_epoch == stamp.scope_epoch + }) + .map(|parent| (render_owner(&parent.owner), parent.scope_ref.clone())); let (owner, kind, reference, epoch) = match stamp.kind { ScopeKind::Head => (stamp.owner, "head", stamp.scope_ref, stamp.scope_epoch), ScopeKind::Worker => (stamp.owner, "worker", stamp.scope_ref, stamp.scope_epoch), @@ -124,6 +134,9 @@ impl State { if coverage == Coverage::AgentAndWorkers && kind != "head" { return invalid(); } + if self.store_error { + return refusal("store_error", "transient"); + } let matches = match ¶ms.target { Target::ApiKey(target) => { principal == "aft" @@ -138,20 +151,27 @@ impl State { if !matches { return not_found(); } - if self.store_error { - return refusal("store_error", "transient"); - } - let owner = match owner { - subc_protocol::Principal::Reserved { module_id } => format!("reserved:{module_id}"), - subc_protocol::Principal::Direct => "direct".into(), - subc_protocol::Principal::Unverified => "unverified".into(), - }; - let key = format!( - "{}:{owner}:{kind}:{reference}:{}", - params.name, - coverage.as_str() + let owner = render_owner(&owner); + let key = ( + params.name.clone(), + owner.clone(), + kind.into(), + reference.clone(), + coverage.as_str().into(), ); - if self.approved.contains(&key) { + let covered = self.approved.iter().any( + |(name, grant_owner, grant_kind, grant_ref, grant_coverage)| { + name == ¶ms.name + && ((grant_owner == &owner && grant_kind == kind && grant_ref == &reference) + || (kind == "worker" + && grant_kind == "head" + && grant_coverage == "agent_and_workers" + && worker_head.as_ref().is_some_and(|(owner, reference)| { + owner == grant_owner && reference == grant_ref + }))) + }, + ); + if covered { return json!({"result":golden(3)}); } if self.setup_incomplete { @@ -160,7 +180,16 @@ impl State { if let Some(pending) = self.pending.get(&key) { return json!({"result":pending}); } - if self.other_pending_in_scope >= 8 || self.other_pending_total + self.pending.len() >= 64 { + let scope_pending = self + .pending + .keys() + .filter(|(_, pending_owner, pending_kind, pending_ref, _)| { + pending_owner == &owner && pending_kind == kind && pending_ref == &reference + }) + .count(); + if self.other_pending_in_scope + scope_pending >= 8 + || self.other_pending_total + self.pending.len() >= 64 + { return refusal("pending_limit", "transient"); } let mut pending = golden(5); @@ -185,6 +214,14 @@ impl State { } } +fn render_owner(owner: &subc_protocol::Principal) -> String { + match owner { + subc_protocol::Principal::Reserved { module_id } => format!("reserved:{module_id}"), + subc_protocol::Principal::Direct => "direct".into(), + subc_protocol::Principal::Unverified => "unverified".into(), + } +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct Params { @@ -297,7 +334,7 @@ impl FakeClaustrum { let state = Arc::clone(&served); let key = key.clone(); tasks.spawn(async move { - subc_transport::authenticate_server(&mut stream, &key, &daemon_id, "0.39.0", Duration::from_secs(2)).await.unwrap(); + if subc_transport::authenticate_server(&mut stream, &key, &daemon_id, "0.39.0", Duration::from_secs(2)).await.is_err() { return; } let mut principal = String::new(); while let Ok(Some(request)) = subc_transport::read_frame(&mut stream).await { if request.header.ty == FrameType::Ping { diff --git a/crates/aft/src/secret/transport_tests.rs b/crates/aft/src/secret/transport_tests.rs index cd74df720f..833803698d 100644 --- a/crates/aft/src/secret/transport_tests.rs +++ b/crates/aft/src/secret/transport_tests.rs @@ -114,6 +114,32 @@ fn secret_use_route_maps_all_sender_refusals_without_releasing_material() { } } +#[test] +fn secret_use_fake_grant_lookup_is_independent_of_requested_coverage_and_covers_linked_workers() { + let daemon = FakeClaustrum::start(); + let transport = transport(&daemon); + let mut head = golden(0); + head["coverage"] = json!("agent_and_workers"); + let pending = call(&transport, head); + daemon + .state + .lock() + .unwrap() + .approve(pending["result"]["statement_b64url"].as_str().unwrap()); + assert_eq!(call(&transport, golden(1)), json!({"result":golden(3)})); + let mut worker = golden(0); + worker["stamp"] = golden(2)["stamp"].clone(); + assert_eq!( + call(&transport, worker.clone()), + json!({"result":golden(3)}) + ); + worker["stamp"]["parent_state"] = json!("ended"); + assert!(matches!( + parse_answer(&call(&transport, worker)).unwrap(), + SecretUseAnswer::Reply(SecretUseReply::ApprovalRequired(_)) + )); +} + #[test] fn secret_use_deadline_covers_scoped_open_reply_and_disconnect_without_secret_errors() { for phase in ["open", "reply", "disconnect", "no-daemon"] { diff --git a/crates/aft/src/secret/wire_tests.rs b/crates/aft/src/secret/wire_tests.rs index aa996e6f4a..cd1ae2eeb2 100644 --- a/crates/aft/src/secret/wire_tests.rs +++ b/crates/aft/src/secret/wire_tests.rs @@ -18,8 +18,9 @@ const VECTORS: &str = include_str!("../../tests/fixtures/secret/secret-approval- /// The sha256 the campaign pinned for the vector file. const VECTORS_SHA256: &str = "efd4ba60641586e143bce81087e4f9f5aaf8a99ffede74190303da7d4495cc27"; -// The six agent-sender request/result examples in claustrum's wire-contract v1 -// §15, in document order. Keep their literal JSON bytes, including statement bytes. +// The six agent-sender request/result examples in claustrum's +// docs/wire-contract-v1.md, §15 "secret.use: the sender contract", in document +// order. Keep their literal JSON bytes, including statement bytes. pub(super) const USE_GOLDENS: &str = include_str!("../../tests/fixtures/secret/secret-use-v1.jsonl"); From 737487d704bff8b8cc7f0d3cc17d42938fbd0b79 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:10:33 +0100 Subject: [PATCH 036/163] mason: reshape vault calls around concrete outgoing requests Checkpoint the approved contract change: bash launch no longer resolves keys or blocks on consent; request-addressed releases replace synthetic grant handles and expiries. Fetch asks per hop. Declined and expired phone cards no longer retry as if approved. Test callers are being migrated to the new seam in the next step. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 95a28a40fcefc018717e592384361a6b37b5f3bf) (cherry picked from commit 452999b6f19948ff00bf7f206e0bf3b1dc0eff9e) --- crates/aft/src/bash_background/registry.rs | 9 + crates/aft/src/bash_background/secrets.rs | 41 +--- crates/aft/src/commands/url_resolution.rs | 29 --- crates/aft/src/secret/consent.rs | 19 +- crates/aft/src/secret/mod.rs | 22 +- crates/aft/src/secret/outcome.rs | 31 +-- crates/aft/src/secret/requests.rs | 184 +++++++++++++++++ crates/aft/src/secret/scope.rs | 45 +---- crates/aft/src/secret/transport.rs | 221 ++++++++++++++------- crates/aft/src/secret/transport_fake.rs | 2 + crates/aft/src/secret/transport_tests.rs | 4 +- crates/aft/src/secret/vault.rs | 193 +++--------------- crates/aft/src/secret/vault/fake.rs | 113 +++++++---- crates/aft/src/secret/wire.rs | 7 + crates/aft/src/secret_proxy/audit.rs | 59 +++--- crates/aft/src/secret_proxy/mod.rs | 52 ++--- crates/aft/src/secret_proxy/request.rs | 167 ++++++++-------- crates/aft/src/subc/consent.rs | 1 + crates/aft/src/subc/mod.rs | 3 +- crates/aft/src/subc/secret_consent.rs | 144 +++++++------- crates/aft/src/url_fetch.rs | 126 +++++------- crates/secret-rules/src/location.rs | 5 +- 22 files changed, 767 insertions(+), 710 deletions(-) create mode 100644 crates/aft/src/secret/requests.rs diff --git a/crates/aft/src/bash_background/registry.rs b/crates/aft/src/bash_background/registry.rs index da4aed719d..b7b76b6078 100644 --- a/crates/aft/src/bash_background/registry.rs +++ b/crates/aft/src/bash_background/registry.rs @@ -3140,6 +3140,15 @@ impl BgTaskRegistry { } append_remote_report(&mut snapshot.output_preview, &state.metadata); } + #[cfg(unix)] + if let Some(runtime) = self.inner.secret_proxies.lock().get(&task.task_id).cloned() { + for line in runtime.lock().approval_status() { + if !snapshot.output_preview.is_empty() { + snapshot.output_preview.push('\n'); + } + snapshot.output_preview.push_str(&line); + } + } snapshot } diff --git a/crates/aft/src/bash_background/secrets.rs b/crates/aft/src/bash_background/secrets.rs index 45f67d9c8a..1369855b09 100644 --- a/crates/aft/src/bash_background/secrets.rs +++ b/crates/aft/src/bash_background/secrets.rs @@ -19,7 +19,7 @@ use crate::sandbox_spawn::SpawnPlan; use crate::sandbox_spawn::{RequestedSandboxTier, SandboxRequirement, SandboxTaskKind}; use crate::secret::{SecretVault, UnavailableVault}; #[cfg(unix)] -use crate::secret_proxy::{GrantedLabel, ProxyConfig, SecretProxy}; +use crate::secret_proxy::{ProxyConfig, SecretProxy}; pub(crate) const REMOTE_REFUSED: &str = "Secret placeholders stay local; remote commands cannot use them."; @@ -62,7 +62,10 @@ pub(crate) fn vault() -> Arc { if let Some(vault) = TEST_VAULT.with(|slot| slot.borrow().clone()) { return vault; } - Arc::new(UnavailableVault) + match crate::secret::transport::for_current_command() { + Some(transport) => Arc::new(crate::secret::SubcSecretVault::new(Arc::new(transport))), + None => Arc::new(UnavailableVault), + } } pub(crate) fn refusal(id: &str, labels: &[String], code: &str, message: &str) -> Response { @@ -155,6 +158,9 @@ pub(crate) struct Runtime { #[cfg(unix)] impl Runtime { + pub(crate) fn approval_status(&self) -> Vec { + self.proxy.approval_status() + } #[cfg(test)] pub(crate) fn address(&self) -> std::net::SocketAddr { self.proxy.address() @@ -244,40 +250,15 @@ pub(crate) fn authorize( ) })?; let vault = vault(); - let mut grants = Vec::new(); - for label in labels { - match crate::subc::resolve_current_secret_label(vault.as_ref(), label) { - crate::subc::LaunchDecision::Proceed { - mut rule, - grant_id, - expires_at_ms, - } => { - rule.grant_id = grant_id; - grants.push(GrantedLabel { - rule, - expires_at_ms, - }); - } - crate::subc::LaunchDecision::Refuse(outcome) => { - return Err(outcome_response(id, labels, outcome)) - } - crate::subc::LaunchDecision::Pending(pending) => { - return Err(refusal( - id, - labels, - crate::secret::SECRET_ACCESS_PENDING, - pending.message, - )) - } - } - } let task = preflight.task.as_ref().expect("preflight owns layout"); let proxy = start_proxy(ProxyConfig { vault, authorization_scope: scopes.authorization.clone(), lifetime_scope: scopes.lifetime.clone(), scanned_labels: labels.to_vec(), - grants, + consent_context: crate::subc::current_secret_consent_context(), + #[cfg(test)] + test_https_port: None, origin: task.paths.task_id.clone(), }) .map_err(|_| refusal(id, labels, "secrets_unavailable", PROXY_UNAVAILABLE))?; diff --git a/crates/aft/src/commands/url_resolution.rs b/crates/aft/src/commands/url_resolution.rs index 21fa338df3..ec82565488 100644 --- a/crates/aft/src/commands/url_resolution.rs +++ b/crates/aft/src/commands/url_resolution.rs @@ -160,7 +160,6 @@ fn resolve_secret_fetch( }); let mut authorization = None; - let mut rules = Vec::with_capacity(labels.len()); if let Some(first) = labels.first() { let scopes = match crate::subc::current_call_scopes() { Ok((scopes, _)) => scopes, @@ -174,33 +173,6 @@ fn resolve_secret_fetch( )); } }; - for label in labels { - match crate::subc::resolve_current_secret_label(vault.as_ref(), label) { - crate::subc::LaunchDecision::Proceed { - mut rule, grant_id, .. - } => { - rule.grant_id = grant_id; - rules.push(rule); - } - crate::subc::LaunchDecision::Refuse(outcome) => { - return Err(secret_outcome_refusal( - &req.id, - vault.as_ref(), - labels, - outcome, - )); - } - crate::subc::LaunchDecision::Pending(pending) => { - return Err(secret_fetch_refusal( - &req.id, - vault.as_ref(), - labels, - crate::secret::SECRET_ACCESS_PENDING, - pending.message, - )); - } - } - } authorization = Some(scopes.authorization); } @@ -213,7 +185,6 @@ fn resolve_secret_fetch( vault: vault.as_ref(), authorization, labels: labels.to_vec(), - rules, call_id: req.id.clone(), }; let document = crate::url_fetch::fetch_url_with_secrets(file, &document_dir, options, &secrets) diff --git a/crates/aft/src/secret/consent.rs b/crates/aft/src/secret/consent.rs index 233e1bfd58..207a36ff73 100644 --- a/crates/aft/src/secret/consent.rs +++ b/crates/aft/src/secret/consent.rs @@ -47,9 +47,11 @@ impl std::fmt::Debug for SecretConsentRequest { /// What happened to a filed secret consent. #[derive(Clone, PartialEq, Eq)] pub enum ConsentOutcome { - /// Cingulate answered. Whatever it said, the caller must resolve again; - /// the answer alone grants nothing. + /// Cingulate allowed this request. Retry the vault once; only its release + /// authorizes substitution. Answered, + Declined, + Expired, /// Still pending when the call's hold ended: the call ends /// `access_pending` and the retried call asks again. Pending, @@ -65,6 +67,8 @@ impl std::fmt::Debug for ConsentOutcome { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ConsentOutcome::Answered => f.write_str("Answered"), + ConsentOutcome::Declined => f.write_str("Declined"), + ConsentOutcome::Expired => f.write_str("Expired"), ConsentOutcome::Pending => f.write_str("Pending"), ConsentOutcome::Unavailable { detail } => f .debug_struct("Unavailable") @@ -84,6 +88,11 @@ pub trait SecretConsentTransport: Send + Sync { /// File `request` and wait for the answer, or for the hold to end. fn file(&self, request: &SecretConsentRequest) -> ConsentOutcome; + /// File once and hold a proxy request for the full approval window. + fn hold(&self, request: &SecretConsentRequest) -> ConsentOutcome { + self.file(request) + } + /// Settle a pending ask the call left behind. fn withhold(&self, request: &SecretConsentRequest) -> ConsentOutcome; } @@ -123,6 +132,12 @@ pub fn file_consent(request: &SecretConsentRequest) -> ConsentOutcome { } } +pub(crate) fn hold_consent(request: &SecretConsentRequest) -> ConsentOutcome { + transport().map_or(ConsentOutcome::Declined, |transport| { + transport.hold(request) + }) +} + /// Settle a pending secret ask. pub fn withhold_consent(request: &SecretConsentRequest) -> ConsentOutcome { match transport() { diff --git a/crates/aft/src/secret/mod.rs b/crates/aft/src/secret/mod.rs index fdf0d41db4..21a30b7573 100644 --- a/crates/aft/src/secret/mod.rs +++ b/crates/aft/src/secret/mod.rs @@ -1,22 +1,20 @@ //! The secret vault seam and its consent adapter. //! -//! [`SecretVault`] is the only way AFT asks for a label's rule or key bytes: -//! `resolve` says whether and where a label may be used, `material` releases the -//! bytes for one request, and `audit` collects one record per decided label. +//! [`SecretVault`] asks for material using each outgoing request's real target. //! [`wire`] holds the `secret.use` contract's request and reply shapes (v1 §15); -//! production wires [`UnavailableVault`], which resolves nothing because the -//! live vault is not deployed and AFT holds no use grant yet. The fake +//! Standalone processes use [`UnavailableVault`]. The fake //! lives behind `cfg(test)` or the `secret-test-fake` feature, which no release //! profile enables. //! //! An eligible call's two scopes ([`eligibility::CallScopes`]) are stamped by -//! the daemon: the authorization scope reaches `resolve`, `material`, the +//! the daemon: the authorization scope reaches `use_secret`, the //! consent ask and the audit record, while the lifetime scope decides when a //! proxy and its matchers stop. pub mod consent; pub mod eligibility; pub mod outcome; +pub(crate) mod requests; pub mod scope; pub mod transport; pub mod vault; @@ -28,14 +26,14 @@ pub use consent::{ }; pub use eligibility::{CallScopes, EligibilityForm, Ineligibility}; pub use outcome::{ - first_failure, outcome_for, outcome_for_material, outcome_for_second_resolve, refuse, - refuse_ineligible, LabelOutcome, SECRETS_UNAVAILABLE, SECRET_ACCESS_PENDING, - SECRET_ACCESS_PENDING_MESSAGE, SECRET_ACCESS_UNAVAILABLE, SECRET_REFUSED, + first_failure, outcome_for, outcome_for_second_use, refuse, refuse_ineligible, LabelOutcome, + SECRETS_UNAVAILABLE, SECRET_ACCESS_PENDING, SECRET_ACCESS_PENDING_MESSAGE, + SECRET_ACCESS_UNAVAILABLE, SECRET_REFUSED, }; pub use scope::SecretScope; pub use vault::{ - now_ms, unrecorded_audit_records, zeroize, AuditError, Material, MaterialError, Resolution, - SecretUseTransport, SecretVault, SubcSecretVault, UnavailableVault, + now_ms, unrecorded_audit_records, zeroize, AuditError, Material, SecretUseTransport, + SecretVault, SubcSecretVault, UnavailableVault, UseAnswer, }; pub use wire::{ parse_answer, parse_reply, ApiKeyTarget, ApprovalRequest, Coverage, LoginTarget, @@ -44,7 +42,7 @@ pub use wire::{ }; #[cfg(any(test, feature = "secret-test-fake"))] -pub(crate) use vault::fake::{approval, granted, FakeVault, FakeVaultScript}; +pub(crate) use vault::fake::{approval, granted, FakeReply, FakeVault, FakeVaultScript}; #[cfg(test)] #[path = "tests.rs"] diff --git a/crates/aft/src/secret/outcome.rs b/crates/aft/src/secret/outcome.rs index f6256ba0c6..3f1757647a 100644 --- a/crates/aft/src/secret/outcome.rs +++ b/crates/aft/src/secret/outcome.rs @@ -8,7 +8,7 @@ //! already return for a held or refused call, plus two of its own. use super::eligibility::Ineligibility; -use super::vault::{MaterialError, Resolution}; +use super::vault::UseAnswer; /// The reply code for a launch refused because its scope is not eligible /// . It outranks every vault answer, because an ineligible call never @@ -98,15 +98,14 @@ pub fn refuse_ineligible(label: &str, reason: Ineligibility) -> LabelOutcome { } /// The outcome for one `resolve` answer. -pub fn outcome_for(label: &str, resolution: &Resolution) -> LabelOutcome { +pub fn outcome_for(label: &str, resolution: &UseAnswer) -> LabelOutcome { match resolution { - Resolution::Granted { .. } => LabelOutcome::Proceed, - Resolution::ApprovalRequired(request) => LabelOutcome::FileConsent { + UseAnswer::Released(_) | UseAnswer::NotFound => LabelOutcome::Proceed, + UseAnswer::ApprovalRequired(request) => LabelOutcome::FileConsent { request_id: request.request_id.clone(), statement_b64url: request.statement_b64url.clone(), }, - Resolution::Denied { reason } => refuse(SECRET_REFUSED, label, reason), - Resolution::Unavailable => refuse( + UseAnswer::Unavailable => refuse( SECRETS_UNAVAILABLE, label, "no vault is connected in this build", @@ -117,13 +116,13 @@ pub fn outcome_for(label: &str, resolution: &Resolution) -> LabelOutcome { /// The outcome after Cingulate answered but the vault still has not granted /// . Cingulate's answer alone never grants: the caller resolves again, and /// this is what a second pending or refused answer leaves. -pub fn outcome_for_second_resolve(label: &str, resolution: &Resolution) -> LabelOutcome { +pub fn outcome_for_second_use(label: &str, resolution: &UseAnswer) -> LabelOutcome { match resolution { // The vault answered at resolution time, which a re-resolve turns into // either a grant or a refusal, never a second pending ask: a second // `ApprovalRequired` means the vault is still waiting and nothing may // be substituted this run. - Resolution::ApprovalRequired(_) => refuse( + UseAnswer::ApprovalRequired(_) => refuse( SECRETS_UNAVAILABLE, label, "the vault is still waiting after the consent answer", @@ -132,22 +131,6 @@ pub fn outcome_for_second_resolve(label: &str, resolution: &Resolution) -> Label } } -/// The outcome for a `material` call that failed at request time. -/// -/// A lapsed grant is a refusal with no consent filed, because the proxy is -/// already live and a fresh consent would not make this request legal. -pub fn outcome_for_material(label: &str, error: &MaterialError) -> LabelOutcome { - match error { - MaterialError::GrantLapsed => refuse( - SECRET_REFUSED, - label, - "the grant expired or its scope ended", - ), - MaterialError::Unavailable { detail } => refuse(SECRETS_UNAVAILABLE, label, detail), - MaterialError::Refused { reason } => refuse(SECRET_REFUSED, label, reason), - } -} - /// The first failing outcome across every scanned label, in scan order. /// /// `None` means every label may proceed. A pending label is a failure like any diff --git a/crates/aft/src/secret/requests.rs b/crates/aft/src/secret/requests.rs new file mode 100644 index 0000000000..cddf1ab25e --- /dev/null +++ b/crates/aft/src/secret/requests.rs @@ -0,0 +1,184 @@ +//! Serialize approval waits for a label without caching any released material. + +use std::collections::{BTreeMap, HashMap}; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, LazyLock, Mutex, Weak}; +use std::time::Duration; + +use super::{ApiKeyTarget, SecretConsentRequest, SecretScope, SecretVault, UseAnswer}; + +/// The phone's approval window is independent of upstream network deadlines. +pub(crate) trait ApprovalClock: Send + Sync { + fn now_ms(&self) -> u64; + fn deadline(&self) -> Pin + Send>>; +} + +struct WallClock; +impl ApprovalClock for WallClock { + fn now_ms(&self) -> u64 { + super::vault::now_ms() + } + fn deadline(&self) -> Pin + Send>> { + Box::pin(tokio::time::sleep(Duration::from_secs(600))) + } +} + +#[derive(Default)] +struct History { + filed: Option<(String, u64)>, + used_after_card: bool, +} +type Slot = tokio::sync::Mutex; +type SlotKey = (SecretScope, String); +static SLOTS: LazyLock>>> = LazyLock::new(Mutex::default); + +pub(crate) struct RequestSecrets { + vault: Arc, + scope: SecretScope, + context: Option<(std::path::PathBuf, String)>, + slots: Mutex>>, + status: Mutex>>, + clock: Arc, +} + +impl RequestSecrets { + pub(crate) fn new( + vault: Arc, + scope: SecretScope, + context: Option<(std::path::PathBuf, String)>, + ) -> Self { + Self { + vault, + scope, + context, + slots: Mutex::default(), + status: Mutex::default(), + clock: Arc::new(WallClock), + } + } + + fn slot(&self, label: &str) -> Arc { + let mut local = self + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + local + .entry(label.into()) + .or_insert_with(|| { + let mut slots = SLOTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + slots.retain(|_, slot| slot.strong_count() > 0); + let key = (self.scope.clone(), label.to_owned()); + if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) { + return slot; + } + let slot = Arc::new(Slot::default()); + slots.insert(key, Arc::downgrade(&slot)); + slot + }) + .clone() + } + + pub(crate) fn status(&self) -> Vec { + self.status + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .values() + .flatten() + .cloned() + .collect() + } + + pub(crate) async fn use_for_request( + &self, + label: &str, + target: &ApiKeyTarget, + ) -> Result { + let result = tokio::select! { + result = self.use_serial(label, target) => result, + () = self.clock.deadline() => Err(()), + }; + self.status + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(label); + result + } + + async fn ask(&self, label: &str, target: &ApiKeyTarget) -> UseAnswer { + let (vault, scope, label, target) = ( + self.vault.clone(), + self.scope.clone(), + label.to_owned(), + target.clone(), + ); + let (send, receive) = tokio::sync::oneshot::channel(); + std::thread::spawn(move || { + let _ = send.send(vault.use_secret(&scope, &label, &target)); + }); + receive.await.unwrap_or(UseAnswer::Unavailable) + } + + async fn use_serial(&self, label: &str, target: &ApiKeyTarget) -> Result { + let slot = self.slot(label); + let mut history = slot.lock().await; + if history + .filed + .as_ref() + .is_some_and(|(_, expires)| *expires <= self.clock.now_ms()) + { + history.filed = None; + } + let answer = self.ask(label, target).await; + let UseAnswer::ApprovalRequired(pending) = answer else { + return Ok(answer); + }; + if history + .filed + .as_ref() + .is_some_and(|(id, _)| id == &pending.request_id) + { + return Err(()); + } + let Some((project_root, session)) = &self.context else { + return Err(()); + }; + let request = SecretConsentRequest { + scope: self.scope.clone(), + project_root: project_root.clone(), + session: session.clone(), + label: label.into(), + request_id: pending.request_id.clone(), + statement_b64url: pending.statement_b64url, + }; + let mut lines = vec![format!("secret {label}: waiting for approval on the phone for {} (requests wait up to 10 minutes)", target.host)]; + if history.used_after_card { + lines.push(format!( + "secret {label}: the approval covered one request; another card was sent" + )); + } + self.status + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(label.into(), lines); + history.filed = Some((pending.request_id.clone(), pending.expires_at_ms)); + let (send, receive) = tokio::sync::oneshot::channel(); + std::thread::spawn(move || { + let _ = send.send(super::consent::hold_consent(&request)); + }); + if receive.await.ok() != Some(super::ConsentOutcome::Answered) { + return Err(()); + } + let answer = self.ask(label, target).await; + match answer { + UseAnswer::ApprovalRequired(_) => Err(()), + UseAnswer::Released(_) => { + history.used_after_card = true; + Ok(answer) + } + _ => Ok(answer), + } + } +} diff --git a/crates/aft/src/secret/scope.rs b/crates/aft/src/secret/scope.rs index 41335a5477..1bf3b1a0dc 100644 --- a/crates/aft/src/secret/scope.rs +++ b/crates/aft/src/secret/scope.rs @@ -9,10 +9,9 @@ use std::sync::Arc; use secret_rules::audit::ScopeRef; -use secret_rules::location::{LabelRule, Location}; use subc_protocol::scope::ScopeStamp; -use super::wire::{ApiKeyTarget, Coverage, Target}; +use super::wire::Coverage; /// The `(owner, kind, ref)` identity of one stamped scope, plus the stamp the /// vault is given. @@ -72,48 +71,6 @@ impl SecretScope { } } - /// The target a resolution call names. - /// - /// The vault matches a target against the record's sealed scope and returns - /// `not_found` on a mismatch, so a resolution has to name the rule's own - /// host. Until a grant exists, that host is unknown: the root path on the - /// default port is the least specific target the contract allows, and a - /// resolution therefore only ever establishes that the label exists and - /// whether it is approved. - pub fn root_target(&self) -> Target { - Target::ApiKey(ApiKeyTarget { - scheme: "https".into(), - host: String::new(), - port: None, - path: "/".into(), - }) - } - - /// The rule a released key's `location` describes, for `label`. - /// - /// The vault's release says where to put the key but not the host or path - /// prefix it was sealed for; those arrive with the grant. A rule built here - /// therefore records the location only, which is what the reply proves. - pub fn rule_for(&self, label: &str, location: &str) -> LabelRule { - secret_rules::location::parse_vault_location(location) - .map(|location| LabelRule { - label: label.to_string(), - host: String::new(), - port: 443, - path_prefix: "/".into(), - location, - grant_id: String::new(), - }) - .unwrap_or(LabelRule { - label: label.to_string(), - host: String::new(), - port: 443, - path_prefix: "/".into(), - location: Location::Bearer, - grant_id: String::new(), - }) - } - /// The coverage a call asks the vault for. A head or worker call asks for /// its own scope; a bound codemode run's authorization scope is its head's, /// so it asks as that head. diff --git a/crates/aft/src/secret/transport.rs b/crates/aft/src/secret/transport.rs index cc4b74c3b5..81a8af00b9 100644 --- a/crates/aft/src/secret/transport.rs +++ b/crates/aft/src/secret/transport.rs @@ -1,6 +1,7 @@ //! Bounded `secret.use` calls over the daemon's attested consumer routes. use std::path::PathBuf; +use std::sync::{Arc, OnceLock}; use std::time::Duration; use serde_json::{json, Value}; @@ -12,15 +13,47 @@ use zeroize::Zeroizing; use super::vault::SecretUseTransport; const UNAVAILABLE: &str = "secret.use transport unavailable"; +static CONNECTION_FILE: std::sync::RwLock> = std::sync::RwLock::new(None); + +pub(crate) fn install_connection_file(path: &std::path::Path) { + *CONNECTION_FILE + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(path.to_owned()); +} + +pub(crate) fn for_current_command() -> Option { + let path = CONNECTION_FILE.read().ok()?.clone()?; + let (_, session) = crate::subc::current_secret_consent_context()?; + SubcSecretUseTransport::new( + path, + BindIdentity::new("aft-secret-use", session), + Duration::from_secs(15), + ) +} /// A transport for a future request-addressed vault client. Construct it at call /// time: a standalone process has no module identity and must not open as direct. -/// No connection or vault availability is cached across calls. +/// One command owns one consumer connection; failed connections are discarded so +/// a later request can reconnect after the daemon returns. #[derive(Clone)] pub struct SubcSecretUseTransport { connection_file: PathBuf, identity: BindIdentity, timeout: Duration, + worker: Arc>>, +} + +struct Work { + params: Value, + principal: ConsumerIdentity, + deadline: tokio::time::Instant, + reply: std::sync::mpsc::Sender>, +} + +struct ConnectionConfig { + connection_file: PathBuf, + identity: BindIdentity, + timeout: Duration, } impl SubcSecretUseTransport { @@ -36,10 +69,16 @@ impl SubcSecretUseTransport { connection_file, identity, timeout, + worker: Arc::default(), }) } - async fn send(&self, params: Value, principal: ConsumerIdentity) -> Result { + async fn send( + config: &ConnectionConfig, + consumer: &subc_client_rs::SubcConsumer, + params: Value, + principal: ConsumerIdentity, + ) -> Result { let target = ¶ms["target"]; if target["scheme"] != "https" || target @@ -54,83 +93,117 @@ impl SubcSecretUseTransport { let scope = ScopeSelector::new(stamp.owner, stamp.scope_ref, Some(stamp.scope_epoch)); let body = serde_json::to_vec(&json!({"method": "secret.use", "params": params})) .map_err(|_| UNAVAILABLE.to_owned())?; - let consumer = crate::fleet_status::connect_subc_consumer( - &self.connection_file, - ConsumerOptions::default(), - ) - .await - .map_err(|_| UNAVAILABLE.to_owned())?; - // Each call owns its consumer, so closing a route cannot interrupt a - // concurrent use of an SDK-cached handle. Dropping the call's runtime on - // timeout also drops the connection and all of its routes. - let result = async { - let route = consumer - .open_route_scoped( - RouteTarget::ManagementSurface { - module_id: "claustrum".into(), - }, - self.identity.clone(), - scope, - CallOptions { - consumer_identity: Some(principal), - timeout: self.timeout, - route_retry_deadline: self.timeout, - ..CallOptions::default() - }, - ) - .await - .map_err(|_| UNAVAILABLE.to_owned())?; - let reply = consumer - .request( - &route, - body, - CallOptions { - timeout: self.timeout, - ..CallOptions::default() - }, - ) - .await; - let result = match reply { - Ok(bytes) => { - let bytes = Zeroizing::new(bytes); - serde_json::from_slice(&bytes).map_err(|_| UNAVAILABLE.to_owned()) - } - // Preserve the distinction between a module Error frame and a - // result refusal. Do not include SDK errors or reply bytes in - // transport diagnostics: either could carry released material. - Err(CallError::Module(error)) => Ok(json!({ - "code": error.code, - "message": error.message, - })), - Err(_) => Err(UNAVAILABLE.to_owned()), - }; - let _ = consumer - .close_handle(&route, subc_client_rs::CloseRouteOptions::default()) - .await; - result + let route = consumer + .open_route_scoped( + RouteTarget::ManagementSurface { + module_id: "claustrum".into(), + }, + config.identity.clone(), + scope, + CallOptions { + consumer_identity: Some(principal), + timeout: config.timeout, + route_retry_deadline: config.timeout, + ..CallOptions::default() + }, + ) + .await + .map_err(|_| UNAVAILABLE.to_owned())?; + let reply = consumer + .request( + &route, + body, + CallOptions { + timeout: config.timeout, + ..CallOptions::default() + }, + ) + .await; + match reply { + Ok(bytes) => { + let bytes = Zeroizing::new(bytes); + serde_json::from_slice(&bytes).map_err(|_| UNAVAILABLE.to_owned()) + } + // Preserve the distinction between a module Error frame and a + // result refusal. Do not include SDK errors or reply bytes in + // transport diagnostics: either could carry released material. + Err(CallError::Module(error)) => Ok(json!({ + "code": error.code, + "message": error.message, + })), + Err(_) => Err(UNAVAILABLE.to_owned()), } - .await; - let _ = consumer.close().await; - result } fn dispatch(&self, params: Value, principal: ConsumerIdentity) -> Result { - let this = self.clone(); - // The seam is synchronous and is also used outside Tokio. A dedicated - // runtime avoids entering a nested runtime on the caller's executor thread. - std::thread::spawn(move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|_| UNAVAILABLE.to_owned())?; - runtime.block_on(async { - tokio::time::timeout(this.timeout, this.send(params, principal)) - .await - .map_err(|_| UNAVAILABLE.to_owned())? + let sender = self.worker.get_or_init(|| { + let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); + let config = ConnectionConfig { + connection_file: self.connection_file.clone(), + identity: self.identity.clone(), + timeout: self.timeout, + }; + std::thread::spawn(move || { + if let Ok(runtime) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + runtime.block_on(Self::serve(config, receiver)); + } + }); + sender + }); + let (reply, answer) = std::sync::mpsc::channel(); + sender + .send(Work { + params, + principal, + deadline: tokio::time::Instant::now() + self.timeout, + reply, }) - }) - .join() - .map_err(|_| UNAVAILABLE.to_owned())? + .map_err(|_| UNAVAILABLE.to_owned())?; + answer + .recv_timeout(self.timeout) + .map_err(|_| UNAVAILABLE.to_owned())? + } + + async fn serve( + config: ConnectionConfig, + mut receiver: tokio::sync::mpsc::UnboundedReceiver, + ) { + let mut consumer = None; + while let Some(work) = receiver.recv().await { + let result = tokio::time::timeout_at(work.deadline, async { + if consumer.is_none() { + consumer = Some( + crate::fleet_status::connect_subc_consumer( + &config.connection_file, + ConsumerOptions::default(), + ) + .await + .map_err(|_| UNAVAILABLE.to_owned())?, + ); + } + Self::send( + &config, + consumer.as_ref().expect("connected consumer"), + work.params, + work.principal, + ) + .await + }) + .await + .unwrap_or_else(|_| Err(UNAVAILABLE.to_owned())); + if result.is_err() { + if let Some(failed) = consumer.take() { + let _ = tokio::time::timeout(Duration::from_secs(1), failed.close()).await; + } + } + let _ = work.reply.send(result); + } + if let Some(consumer) = consumer { + let _ = tokio::time::timeout(Duration::from_secs(1), consumer.close()).await; + } } } diff --git a/crates/aft/src/secret/transport_fake.rs b/crates/aft/src/secret/transport_fake.rs index 79638dc060..3d40f55b6e 100644 --- a/crates/aft/src/secret/transport_fake.rs +++ b/crates/aft/src/secret/transport_fake.rs @@ -37,6 +37,7 @@ fn invalid() -> Value { pub(crate) struct State { pub opens: Vec, pub calls: usize, + pub connections: usize, pub hold_open: bool, pub hold_reply: bool, pub disconnect: bool, @@ -335,6 +336,7 @@ impl FakeClaustrum { let key = key.clone(); tasks.spawn(async move { if subc_transport::authenticate_server(&mut stream, &key, &daemon_id, "0.39.0", Duration::from_secs(2)).await.is_err() { return; } + state.lock().unwrap().connections += 1; let mut principal = String::new(); while let Ok(Some(request)) = subc_transport::read_frame(&mut stream).await { if request.header.ty == FrameType::Ping { diff --git a/crates/aft/src/secret/transport_tests.rs b/crates/aft/src/secret/transport_tests.rs index 833803698d..c752d02d83 100644 --- a/crates/aft/src/secret/transport_tests.rs +++ b/crates/aft/src/secret/transport_tests.rs @@ -14,6 +14,7 @@ fn transport(daemon: &FakeClaustrum) -> SubcSecretUseTransport { connection_file: daemon.path.clone(), identity: BindIdentity::new("aft-secret-use", "secret-call"), timeout: Duration::from_secs(2), + worker: Arc::default(), } } @@ -49,7 +50,8 @@ fn secret_use_route_preserves_the_stamp_and_releases_only_after_phone_approval() assert!(!format!("{key:?}").contains("c2stZXhhbXBsZQ")); let state = daemon.state.lock().unwrap(); assert_eq!(state.calls, 3); - assert_eq!(state.opens.len(), 3); + assert_eq!(state.connections, 1); + assert_eq!(state.opens.len(), 1); for open in &state.opens { assert_eq!( open["consumer_identity"], diff --git a/crates/aft/src/secret/vault.rs b/crates/aft/src/secret/vault.rs index 49b251ae4c..5b474204bf 100644 --- a/crates/aft/src/secret/vault.rs +++ b/crates/aft/src/secret/vault.rs @@ -1,30 +1,14 @@ -//! The vault seam: the one way AFT asks for a label's rule and key material. -//! -//! AFT never reads a key itself. The vault maps a label to a rule in a -//! `secret.use` reply ([`crate::secret::wire`]), hands over the bytes for one -//! request or fetch hop, and collects one audit record per decided label. -//! -//! `material` exists because a grant is a permission, not a promise of the same -//! bytes: the vault releases material per request, and AFT must not cache it. -//! Key bytes live in a zeroizing buffer whose `Debug` prints a length, so no log -//! line or panic message carries a key. -//! -//! The live vault is not deployed and AFT holds no `secret.use` grant yet, so -//! production wires [`UnavailableVault`], which resolves nothing and releases -//! nothing. [`SubcSecretVault`] is the real client, built and tested -//! against the contract's golden examples; it becomes the production stand-in's -//! replacement once the daemon can carry the call. The fake for tests compiles -//! only under `cfg(test)` or the `secret-test-fake` feature, which no release -//! profile enables (`scripts/check-secret-test-fake.sh` checks that in CI). +//! Request-addressed vault access. Every outgoing use names its real target; +//! material is zeroized after that request and is never cached between uses. use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use secret_rules::audit::AuditRecordV1; -use secret_rules::location::LabelRule; use super::wire::{ - self, ApprovalRequest, SecretUseAnswer, SecretUseReply, SecretUseRequest, VaultError, + self, ApiKeyTarget, ApprovalRequest, ReleasedApiKey, SecretUseAnswer, SecretUseReply, + SecretUseRequest, }; /// Key bytes for one request or fetch hop. @@ -83,47 +67,19 @@ pub fn zeroize(bytes: &mut [u8]) { std::sync::atomic::compiler_fence(Ordering::SeqCst); } -/// What the vault says about one label. -#[derive(Clone, PartialEq, Eq, Debug)] -pub enum Resolution { - /// A rule to follow and the grant that allows material for it. - Granted { - /// Where and how the placeholder may be replaced. - rule: LabelRule, - /// Opaque grant identifier passed back to `material`. - grant_id: String, - /// When the grant lapses, in milliseconds since the epoch. - expires_at_ms: u64, - }, +/// What the vault says about one concrete outgoing request. +#[derive(Debug)] +pub enum UseAnswer { + /// Key and location for this request only. + Released(ReleasedApiKey), /// The vault is waiting for the user's answer. ApprovalRequired(ApprovalRequest), - /// The vault refused the label. - Denied { - /// The vault's reason, plain text with no key material. - reason: String, - }, - /// No vault answers. Production answers this until the vault ships and AFT - /// holds a grant, which keeps the seam fail-closed. + /// Uniform refusal: forwarding the literal placeholder reveals no key. + NotFound, + /// Transport, grammar, setup, and store failures never permit substitution. Unavailable, } -/// Why `material` handed out no bytes. -#[derive(Clone, PartialEq, Eq, Debug)] -pub enum MaterialError { - /// The grant lapsed: its expiry passed, it was revoked, or its scope ended. - GrantLapsed, - /// The vault could not be reached, or answered transiently. - Unavailable { - /// Plain-text detail with no key material. - detail: String, - }, - /// The vault refused to release material for the grant. - Refused { - /// The vault's reason, plain text with no key material. - reason: String, - }, -} - /// Why `audit` did not take a record. #[derive(Clone, PartialEq, Eq, Debug)] pub enum AuditError { @@ -140,23 +96,19 @@ pub enum AuditError { }, } -/// The three calls AFT makes on the vault. +/// Request-addressed release and local audit reporting. /// /// Both scopes are [`crate::secret::SecretScope`] values the daemon stamped, /// never tool arguments or session ids. pub trait SecretVault: Send + Sync { - /// Ask which rule a label follows, without asking for the key bytes. - fn resolve(&self, scope: &crate::secret::SecretScope, label: &str) -> Resolution; - - /// Fetch the bytes for one request or fetch hop whose target matched the - /// rule that `grant_id` belongs to. Called once per matching request; the - /// same grant may release different bytes after a rotation. - fn material( + /// Ask once for every outgoing request, including repeated requests after + /// approval. The vault audits uses and may rotate material between calls. + fn use_secret( &self, scope: &crate::secret::SecretScope, label: &str, - grant_id: &str, - ) -> Result; + target: &ApiKeyTarget, + ) -> UseAnswer; /// Hand one record to the vault's audit sink. fn audit(&self, record: &AuditRecordV1) -> Result<(), AuditError>; @@ -170,30 +122,18 @@ pub trait SecretVault: Send + Sync { } } -/// The vault production uses while the live vault is undeployed and AFT holds -/// no `secret.use` grant. -/// -/// Every resolution is `Unavailable` and `material` refuses, so nothing is ever -/// substituted without a real grant, whatever the configuration says. No config -/// file, environment variable or project setting selects another production -/// vault, so this cannot be switched on by mistake. +/// Standalone processes have no attested AFT identity and cannot ask the vault. #[derive(Debug, Default, Clone, Copy)] pub struct UnavailableVault; impl SecretVault for UnavailableVault { - fn resolve(&self, _scope: &crate::secret::SecretScope, _label: &str) -> Resolution { - Resolution::Unavailable - } - - fn material( + fn use_secret( &self, _scope: &crate::secret::SecretScope, _label: &str, - _grant_id: &str, - ) -> Result { - Err(MaterialError::Unavailable { - detail: "no vault is connected".into(), - }) + _target: &ApiKeyTarget, + ) -> UseAnswer { + UseAnswer::Unavailable } fn audit(&self, _record: &AuditRecordV1) -> Result<(), AuditError> { @@ -238,9 +178,7 @@ pub trait SecretUseTransport: Send + Sync { /// The real client: `secret.use` over the daemon's transport (§15). /// /// AFT is one of only two senders the vault admits, and it receives only API -/// keys. `resolve` reads the reply's rule without keeping any bytes the vault -/// may have released; a released key at resolution time is zeroized -/// immediately, exactly as the contract's material rule requires. +/// keys. Each reply belongs to the real target supplied with that call. pub struct SubcSecretVault { transport: std::sync::Arc, } @@ -274,66 +212,21 @@ impl SubcSecretVault { } impl SecretVault for SubcSecretVault { - fn resolve(&self, scope: &crate::secret::SecretScope, label: &str) -> Resolution { - // A resolution asks about every path, so it names the root path: the - // contract matches the target against the record's sealed scope, and a - // released key here is dropped, never kept. - let target = scope.root_target(); - match self.call(scope, label, target) { - Ok(SecretUseAnswer::Reply(SecretUseReply::ReleasedApiKey(key))) => { - let rule = scope.rule_for(label, &key.location); - // The released bytes are not what this call asked for; drop them - // so they are zeroized here rather than retained. - drop(key); - Resolution::Granted { - rule, - grant_id: String::new(), - expires_at_ms: 0, - } - } - Ok(SecretUseAnswer::Reply(SecretUseReply::ReleasedLogin(_))) => Resolution::Denied { - reason: "the vault released a login to a sender that only takes API keys".into(), - }, - Ok(SecretUseAnswer::Reply(SecretUseReply::ApprovalRequired(request))) => { - Resolution::ApprovalRequired(request) - } - Ok(SecretUseAnswer::Refused(error)) => resolution_for_error(&error), - Ok(SecretUseAnswer::Rejected(error)) => { - if error.code == "invalid_params" { - Resolution::Denied { - reason: format!("the vault refused the request: {}", error.code), - } - } else { - Resolution::Unavailable - } - } - Err(_) => Resolution::Unavailable, - } - } - - fn material( + fn use_secret( &self, scope: &crate::secret::SecretScope, label: &str, - _grant_id: &str, - ) -> Result { - let target = scope.root_target(); - match self.call(scope, label, target) { + target: &ApiKeyTarget, + ) -> UseAnswer { + match self.call(scope, label, wire::Target::ApiKey(target.clone())) { Ok(SecretUseAnswer::Reply(SecretUseReply::ReleasedApiKey(key))) => { - Ok(Material::new(key.material().to_vec())) - } - Ok(SecretUseAnswer::Reply(SecretUseReply::ReleasedLogin(_))) => { - Err(MaterialError::Refused { - reason: "the vault released a login, not an API key".into(), - }) + UseAnswer::Released(key) } - Ok(SecretUseAnswer::Reply(SecretUseReply::ApprovalRequired(_))) => { - Err(MaterialError::GrantLapsed) + Ok(SecretUseAnswer::Reply(SecretUseReply::ApprovalRequired(request))) => { + UseAnswer::ApprovalRequired(request) } - Ok(SecretUseAnswer::Refused(error)) => Err(error_for_material(&error)), - Ok(SecretUseAnswer::Rejected(_)) | Err(_) => Err(MaterialError::Unavailable { - detail: "the vault did not answer".into(), - }), + Ok(SecretUseAnswer::Refused(error)) if error.code == "not_found" => UseAnswer::NotFound, + _ => UseAnswer::Unavailable, } } @@ -345,26 +238,6 @@ impl SecretVault for SubcSecretVault { } } -/// Map a result error onto a resolution (the outcome table). -fn resolution_for_error(error: &VaultError) -> Resolution { - match error.code.as_str() { - "not_found" | "invalid_params" => Resolution::Denied { - reason: "the vault does not know this secret or target".into(), - }, - _ => Resolution::Unavailable, - } -} - -/// Map a result error onto a material refusal. -fn error_for_material(error: &VaultError) -> MaterialError { - match error.code.as_str() { - "not_found" => MaterialError::GrantLapsed, - code => MaterialError::Refused { - reason: format!("the vault refused to release the secret: {code}"), - }, - } -} - /// The fake vault, compiled only for tests or an explicitly feature-enabled /// build. #[cfg(any(test, feature = "secret-test-fake"))] diff --git a/crates/aft/src/secret/vault/fake.rs b/crates/aft/src/secret/vault/fake.rs index b13baed12b..2c1978ed28 100644 --- a/crates/aft/src/secret/vault/fake.rs +++ b/crates/aft/src/secret/vault/fake.rs @@ -12,22 +12,31 @@ use std::sync::{Arc, Mutex, MutexGuard}; use secret_rules::audit::AuditRecordV1; use secret_rules::location::LabelRule; -use super::{AuditError, Material, MaterialError, Resolution, SecretVault}; +use super::{AuditError, Material, SecretVault, UseAnswer}; use crate::secret::wire::ApprovalRequest; use crate::secret::SecretScope; +use crate::secret::{ApiKeyTarget, ReleasedApiKey}; /// Everything a test makes the fake say, plus the record of what it was asked. #[derive(Default)] pub struct FakeVaultScript { /// The answer `resolve` gives, per label. A label with no entry answers /// `Unavailable`, which is also what an unconfigured vault says. - pub resolutions: HashMap, + pub answers: HashMap, /// Bytes `material` hands out, per label, taking one entry per call so a /// test can rotate a key under one grant. A label runs out of bytes and /// then repeats the last one. pub material: HashMap>>, } +#[derive(Clone, Debug)] +pub enum FakeReply { + Rule(LabelRule), + ApprovalRequired(ApprovalRequest), + NotFound, + Unavailable, +} + struct FakeVaultState { script: FakeVaultScript, /// One line per `resolve`, in call order. @@ -94,11 +103,8 @@ impl FakeVault { } /// Make `resolve` answer `label` with `resolution`. - pub fn set_resolution(&self, label: &str, resolution: Resolution) { - self.lock() - .script - .resolutions - .insert(label.to_string(), resolution); + pub fn set_answer(&self, label: &str, answer: FakeReply) { + self.lock().script.answers.insert(label.to_string(), answer); } /// The bytes `material` hands out for `label`, in call order. @@ -115,7 +121,7 @@ impl FakeVault { } /// How many times `resolve` was called. - pub fn resolve_calls(&self) -> u64 { + pub fn use_calls(&self) -> u64 { self.resolve_count.load(Ordering::SeqCst) } @@ -145,18 +151,14 @@ impl FakeVault { } /// The next bytes for `label`, honouring rotation. - fn next_material(&self, label: &str) -> Result { + fn next_material(&self, label: &str) -> Option { let mut state = self.lock(); state.materials.push(label.to_string()); let Some(count) = state.script.material.get(label).map(Vec::len) else { - return Err(MaterialError::Refused { - reason: "the fake vault has no material for this label".into(), - }); + return None; }; if count == 0 { - return Err(MaterialError::Refused { - reason: "the fake vault's material list is empty".into(), - }); + return None; } let index = { // Rotation advances per label, so a second request under the same @@ -167,31 +169,54 @@ impl FakeVault { index }; let bytes = state.script.material[label][index].clone(); - Ok(Material::new(bytes)) + Some(Material::new(bytes)) } } impl SecretVault for FakeVault { - fn resolve(&self, _scope: &SecretScope, label: &str) -> Resolution { + fn use_secret(&self, _scope: &SecretScope, label: &str, target: &ApiKeyTarget) -> UseAnswer { self.resolve_count.fetch_add(1, Ordering::SeqCst); - let mut state = self.lock(); - state.resolves.push(label.to_string()); - state - .script - .resolutions - .get(label) - .cloned() - .unwrap_or(Resolution::Unavailable) - } - - fn material( - &self, - _scope: &SecretScope, - label: &str, - _grant_id: &str, - ) -> Result { - self.material_count.fetch_add(1, Ordering::SeqCst); - self.next_material(label) + let answer = { + let mut state = self.lock(); + state.resolves.push(label.to_string()); + state + .script + .answers + .get(label) + .cloned() + .unwrap_or(FakeReply::Unavailable) + }; + match answer { + FakeReply::Rule(rule) => { + let request = secret_rules::target::RequestTarget { + host: target.host.clone(), + port: target.port.unwrap_or(443), + scheme: &target.scheme, + target: &target.path, + }; + if request + .matches(&secret_rules::target::RuleTarget { + host: &rule.host, + port: rule.port, + path_prefix: &rule.path_prefix, + }) + .is_err() + { + return UseAnswer::NotFound; + } + self.material_count.fetch_add(1, Ordering::SeqCst); + match self.next_material(label) { + Some(material) => UseAnswer::Released(ReleasedApiKey::new( + location_text(&rule.location), + material, + )), + None => UseAnswer::Unavailable, + } + } + FakeReply::ApprovalRequired(pending) => UseAnswer::ApprovalRequired(pending), + FakeReply::NotFound => UseAnswer::NotFound, + FakeReply::Unavailable => UseAnswer::Unavailable, + } } fn audit(&self, record: &AuditRecordV1) -> Result<(), AuditError> { @@ -208,12 +233,18 @@ impl SecretVault for FakeVault { } } -/// A granted resolution for tests. -pub fn granted(rule: LabelRule, grant_id: &str) -> Resolution { - Resolution::Granted { - rule, - grant_id: grant_id.to_string(), - expires_at_ms: 0, +/// A matching record for an addressed test call. +pub fn granted(rule: LabelRule) -> FakeReply { + FakeReply::Rule(rule) +} + +pub(crate) fn location_text(location: &secret_rules::location::Location) -> String { + use secret_rules::location::Location; + match location { + Location::Bearer => "authorization_bearer".into(), + Location::Header(name) => format!("header:{name}"), + Location::Query(name) => format!("query:{name}"), + Location::Json(pointer) => format!("json_pointer:{pointer}"), } } diff --git a/crates/aft/src/secret/wire.rs b/crates/aft/src/secret/wire.rs index 2e966010b3..6fc14c4e4c 100644 --- a/crates/aft/src/secret/wire.rs +++ b/crates/aft/src/secret/wire.rs @@ -187,6 +187,13 @@ pub struct ReleasedApiKey { } impl ReleasedApiKey { + pub fn new(location: String, bytes: crate::secret::vault::Material) -> Self { + Self { location, bytes } + } + + pub fn into_parts(self) -> (String, crate::secret::vault::Material) { + (self.location, self.bytes) + } /// The key bytes for the one request that may carry them. pub fn material(&self) -> &[u8] { self.bytes.expose() diff --git a/crates/aft/src/secret_proxy/audit.rs b/crates/aft/src/secret_proxy/audit.rs index 24c4b2c82d..7b41a07de2 100644 --- a/crates/aft/src/secret_proxy/audit.rs +++ b/crates/aft/src/secret_proxy/audit.rs @@ -5,8 +5,8 @@ use super::State; use hyper::body::Incoming; use hyper::Request; use secret_rules::audit::{AuditReason, AuditRecordV1, Decision}; -use secret_rules::location::{location_kind_token, LabelOutcome}; -use secret_rules::target::{ConnectAuthority, RequestTarget, RuleTarget}; +use secret_rules::location::{location_kind_token, LabelOutcome, LabelRule}; +use secret_rules::target::ConnectAuthority; use std::sync::atomic::{AtomicU64, Ordering}; static REQUEST_ID: AtomicU64 = AtomicU64::new(1); @@ -16,6 +16,7 @@ pub(super) struct RequestAudit<'a> { authority: ConnectAuthority, request_id: String, labels: Vec<(Option, String)>, + rules: Vec, done: bool, } @@ -30,35 +31,15 @@ impl<'a> RequestAudit<'a> { .path_and_query() .map(|p| p.as_str()) .unwrap_or("/"); - let view = RequestTarget { - host: authority.host.clone(), - port: authority.port, - scheme: "https", - target, - }; let mut labels = Vec::new(); - for (index, grant) in state.config.grants.iter().enumerate() { - let rule = &grant.rule; - let matched = view - .matches(&RuleTarget { - host: &rule.host, - port: rule.port, - path_prefix: &rule.path_prefix, - }) - .is_ok(); - if matched - || mentions(target.as_bytes(), &rule.label) + for label in &state.config.scanned_labels { + if mentions(target.as_bytes(), label) || request .headers() .values() - .any(|v| mentions(v.as_bytes(), &rule.label)) + .any(|v| mentions(v.as_bytes(), label)) { - labels.push((Some(index), rule.label.clone())); - } - } - if labels.is_empty() { - if let Some(grant) = state.config.grants.first() { - labels.push((Some(0), grant.rule.label.clone())); + labels.push((None, label.clone())); } } Self { @@ -66,23 +47,38 @@ impl<'a> RequestAudit<'a> { authority: authority.clone(), request_id: REQUEST_ID.fetch_add(1, Ordering::Relaxed).to_string(), labels, + rules: Vec::new(), done: false, } } - pub(super) fn decide(&mut self, outcomes: &[LabelOutcome]) -> Result<(), String> { + pub(super) fn decide( + &mut self, + rules: &[LabelRule], + outcomes: &[LabelOutcome], + ) -> Result<(), String> { + self.rules = rules.to_vec(); + for (index, label) in &mut self.labels { + *index = rules.iter().position(|rule| &rule.label == label); + } self.emit(Some(outcomes)) } + pub(super) fn mention(&mut self, label: &str) { + if !self.labels.iter().any(|(_, known)| known == label) { + self.labels.push((None, label.to_owned())); + } + } + pub(super) fn refused_for(&mut self, label: &str) { if !self.done && !label.is_empty() && !self .state .config - .grants + .scanned_labels .iter() - .any(|g| g.rule.label == label) + .any(|known| known == label) { self.labels.clear(); self.labels.push((None, label.to_owned())); @@ -107,6 +103,7 @@ impl<'a> RequestAudit<'a> { (Decision::Literal, AuditReason::NotWholeValue) } Some(_) => (Decision::Literal, AuditReason::OutOfScope), + None if outcomes.is_some() => (Decision::Literal, AuditReason::OutOfScope), None => (Decision::Refused, AuditReason::Refused), }; let mut record = AuditRecordV1::new( @@ -119,7 +116,7 @@ impl<'a> RequestAudit<'a> { ) .with_scope(Some(self.state.config.authorization_scope.to_scope_ref())); if let Some(index) = index { - let rule = &self.state.config.grants[*index].rule; + let rule = &self.rules[*index]; record = record.with_rule( &self.authority.host, self.authority.port, @@ -157,7 +154,7 @@ impl Drop for RequestAudit<'_> { } } -fn mentions(bytes: &[u8], label: &str) -> bool { +pub(super) fn mentions(bytes: &[u8], label: &str) -> bool { let needle = secret_rules::placeholder::placeholder_text(label); for start in 0..bytes.len() { if bytes[start] != b'{' && bytes[start] != b'%' { diff --git a/crates/aft/src/secret_proxy/mod.rs b/crates/aft/src/secret_proxy/mod.rs index ed86d30dff..a4da0ce5e2 100644 --- a/crates/aft/src/secret_proxy/mod.rs +++ b/crates/aft/src/secret_proxy/mod.rs @@ -75,9 +75,9 @@ use hyper::{Method, Request, Response, StatusCode}; use hyper_util::rt::{TokioExecutor, TokioIo}; use parking_lot::Mutex; use rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; -use secret_rules::location::{validate_header_name, LabelRule, Location}; +use secret_rules::location::Location; use secret_rules::scrub::{MatcherSet, Scrubber}; -use secret_rules::target::{canonical_host, ConnectAuthority}; +use secret_rules::target::ConnectAuthority; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; use zeroize::Zeroizing; @@ -90,19 +90,14 @@ pub(crate) type Body = http_body_util::combinators::UnsyncBoxBody, pub authorization_scope: SecretScope, pub lifetime_scope: SecretScope, pub scanned_labels: Vec, - pub grants: Vec, + pub consent_context: Option<(std::path::PathBuf, String)>, + #[cfg(test)] + pub test_https_port: Option, pub origin: String, } @@ -175,6 +170,7 @@ pub(crate) mod test_hardening { } pub(crate) struct State { + approvals: crate::secret::requests::RequestSecrets, config: ProxyConfig, session: AtomicI32, session_birth: std::sync::atomic::AtomicU64, @@ -199,6 +195,13 @@ pub(crate) struct State { } impl State { + fn is_https_port(&self, port: u16) -> bool { + #[cfg(test)] + if self.config.test_https_port == Some(port) { + return true; + } + port == 443 + } fn idle_timeout(&self) -> Duration { #[cfg(test)] { @@ -347,17 +350,11 @@ impl SecretProxy { return Err(ProxyError::UnsupportedPlatform); } let scanned: BTreeSet<_> = config.scanned_labels.iter().collect(); - let resolved: BTreeSet<_> = config.grants.iter().map(|g| &g.rule.label).collect(); if scanned.is_empty() - || scanned != resolved - || resolved.len() != config.grants.len() - || config.grants.iter().any(|g| { - let r = &g.rule; - !secret_rules::placeholder::is_label(&r.label) - || canonical_host(&r.host).is_err() - || secret_rules::target::normalize_path_prefix(&r.path_prefix).map_or(true, |p| p != r.path_prefix) - || matches!(&r.location, Location::Header(name) if validate_header_name(name).is_err()) - }) + || scanned.len() != config.scanned_labels.len() + || scanned + .iter() + .any(|label| !secret_rules::placeholder::is_label(label)) { return Err(ProxyError::InvalidRules); } @@ -384,6 +381,11 @@ impl SecretProxy { Zeroizing::new(base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&random[..])); let expected_auth = basic_credentials(&token)?; let state = Arc::new(State { + approvals: crate::secret::requests::RequestSecrets::new( + config.vault.clone(), + config.authorization_scope.clone(), + config.consent_context.clone(), + ), config, session: AtomicI32::new(0), session_birth: std::sync::atomic::AtomicU64::new(0), @@ -441,6 +443,10 @@ impl SecretProxy { self.address } + pub(crate) fn approval_status(&self) -> Vec { + self.state.approvals.status() + } + /// Exec-time only: never persist the returned credential with a task request. pub fn proxy_url(&self) -> Zeroizing { let address = self.address.to_string(); @@ -572,11 +578,7 @@ async fn connect( }) { return Ok(refused()); } - let intercepted = state - .config - .grants - .iter() - .any(|g| canonical_host(&g.rule.host).ok().as_ref() == Some(&authority.host)); + let intercepted = state.is_https_port(authority.port); let tls = if intercepted { match state.tls_for(&authority.host) { Ok(tls) => Some(tls), diff --git a/crates/aft/src/secret_proxy/request.rs b/crates/aft/src/secret_proxy/request.rs index 92d60e1bda..ad0851648d 100644 --- a/crates/aft/src/secret_proxy/request.rs +++ b/crates/aft/src/secret_proxy/request.rs @@ -10,10 +10,7 @@ use secret_rules::location::{ evaluate_request_with_connect, HeaderField, LabelOutcome, Location, RequestView, }; use secret_rules::material::{check_admissible, encode_for_location}; -use secret_rules::target::{ - authority_matches_connect, normalize_request_target, ConnectAuthority, RequestTarget, - RuleTarget, -}; +use secret_rules::target::{authority_matches_connect, normalize_request_target, ConnectAuthority}; use zeroize::Zeroizing; use super::{full, refusal, Body, State}; @@ -36,9 +33,9 @@ pub(super) async fn forward( .is_some_and(|v| v.as_bytes().eq_ignore_ascii_case(b"websocket")); let fallback = state .config - .grants + .scanned_labels .first() - .map(|g| g.rule.label.as_str()) + .map(String::as_str) .unwrap_or(""); let result = if let Some(lease) = &lease { prepare_and_send(request, &authority, &state, lease.clone(), &mut audit).await @@ -120,28 +117,8 @@ async fn prepare_and_send( .is_some_and(|v| v.as_bytes().eq_ignore_ascii_case(b"websocket")); let client_upgrade = ws.then(|| hyper::upgrade::on(&mut request)); let (mut parts, mut incoming) = request.into_parts(); - let rules: Vec<_> = state.config.grants.iter().map(|g| g.rule.clone()).collect(); - let target_view = RequestTarget { - host: authority.host.clone(), - port: authority.port, - scheme: "https", - target: &target, - }; - let matches: Vec<_> = rules - .iter() - .map(|r| { - target_view - .matches(&RuleTarget { - host: &r.host, - port: r.port, - path_prefix: &r.path_prefix, - }) - .is_ok() - }) - .collect(); - // A runtime-built header label is not a launch grant. Refuse it before - // contacting the vault; other destinations and non-header text are literal. - if matches.iter().any(|matched| *matched) { + // Runtime-built labels cannot widen the labels scanned at launch. + { for value in parts.headers.values() { for (offset, byte) in value.as_bytes().iter().enumerate() { if *byte != b'{' { @@ -150,19 +127,29 @@ async fn prepare_and_send( if let Some(placeholder) = secret_rules::placeholder::parse_at(value.as_bytes(), offset) { - if !rules.iter().any(|rule| rule.label == placeholder.label) { + if !state + .config + .scanned_labels + .iter() + .any(|label| label == placeholder.label) + { return Err(placeholder.label.to_owned()); } } } } } - let has_json = rules - .iter() - .zip(&matches) - .any(|(r, m)| *m && matches!(r.location, Location::Json(_))); - // Only JSON substitution needs a buffered body. Other uploads remain - // streaming; they are never searched for placeholders outside their rule. + // A JSON deposit is not known until the addressed release. Buffer bounded + // JSON requests so their placeholders can be discovered before asking. + let has_json = parts + .headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| { + v.split(';') + .next() + .is_some_and(|v| v.trim().eq_ignore_ascii_case("application/json")) + }); let mut body = body_buffer(has_json); if has_json { while let Some(frame) = incoming.frame().await { @@ -185,6 +172,56 @@ async fn prepare_and_send( } } } + let path = target.split(['?', '#']).next().unwrap_or("/"); + let path = normalize_request_target(path).map_err(|_| fail())?; + let addressed = crate::secret::ApiKeyTarget { + scheme: "https".into(), + host: authority.host.clone(), + port: Some(443), + path: String::from_utf8(path.wire).map_err(|_| fail())?, + }; + let mut rules = Vec::new(); + let mut materials = Vec::new(); + for label in &state.config.scanned_labels { + if !super::audit::mentions(target.as_bytes(), label) + && !parts + .headers + .values() + .any(|value| super::audit::mentions(value.as_bytes(), label)) + && !super::audit::mentions(&body, label) + { + continue; + } + audit.mention(label); + if !state.is_https_port(authority.port) { + continue; + } + let answer = match state.approvals.use_for_request(label, &addressed).await { + Ok(answer) => answer, + Err(()) => return Ok(not_approved(label, &authority.host)), + }; + match answer { + crate::secret::UseAnswer::NotFound => continue, + crate::secret::UseAnswer::Released(released) => { + let (location, material) = released.into_parts(); + let location = secret_rules::location::parse_vault_location(&location) + .map_err(|_| label.clone())?; + check_admissible(material.expose(), label, &location).map_err(|_| label.clone())?; + state + .install(label, material.expose()) + .map_err(|_| label.clone())?; + rules.push(secret_rules::location::LabelRule { + label: label.clone(), + host: addressed.host.clone(), + port: authority.port, + path_prefix: addressed.path.clone(), + location, + }); + materials.push(Some(material)); + } + _ => return Err(label.clone()), + } + } let headers = parts .headers .iter() @@ -203,53 +240,19 @@ async fn prepare_and_send( } if has_json { let json: serde_json::Value = serde_json::from_slice(&body).map_err(|_| fail())?; - for (rule, matched) in rules.iter().zip(&matches) { + for rule in &rules { if let Location::Json(pointer) = &rule.location { - if *matched && json.pointer(pointer).is_some_and(|v| !v.is_string()) { + if json.pointer(pointer).is_some_and(|v| !v.is_string()) { return Err(rule.label.clone()); } } } } - for (grant, matched) in state.config.grants.iter().zip(&matches) { - if *matched && crate::secret::vault::now_ms() >= grant.expires_at_ms { - return Err(grant.rule.label.clone()); - } - } - // Authenticate the upstream first. Material is the vault's revocation - // check, so obtain it only after asynchronous connection work ends. + // No HTTP request reaches the upstream until every required approval ends. let upstream = connect_upstream(authority, state, ws).await?; let http2 = matches!(&upstream, Upstream::Http2(_)); let response = { let _gate = state.material_gate.lock(); - let mut materials = Vec::new(); - for ((rule, matched), grant) in rules.iter().zip(&matches).zip(&state.config.grants) { - if !matched { - materials.push(None); - continue; - } - if state.stopped.load(Ordering::Acquire) - || !state.generation_valid() - || crate::secret::vault::now_ms() >= grant.expires_at_ms - { - return Err(rule.label.clone()); - } - let material = state - .config - .vault - .material( - &state.config.authorization_scope, - &rule.label, - &rule.grant_id, - ) - .map_err(|_| rule.label.clone())?; - check_admissible(material.expose(), &rule.label, &rule.location) - .map_err(|_| rule.label.clone())?; - state - .install(&rule.label, material.expose()) - .map_err(|_| rule.label.clone())?; - materials.push(Some(material)); - } let mut query_edits = Vec::new(); let mut body_edits = Vec::new(); let mut header_edits = Vec::new(); @@ -352,20 +355,10 @@ async fn prepare_and_send( if state.stopped.load(Ordering::Acquire) || !state.generation_valid() { return Err(fail()); } - for (grant, matched) in state.config.grants.iter().zip(&matches) { - if *matched && crate::secret::vault::now_ms() >= grant.expires_at_ms { - return Err(grant.rule.label.clone()); - } - } - audit.decide(&plan.labels)?; + audit.decide(&rules, &plan.labels)?; if state.stopped.load(Ordering::Acquire) || !state.generation_valid() { return Err(fail()); } - for (grant, matched) in state.config.grants.iter().zip(&matches) { - if *matched && crate::secret::vault::now_ms() >= grant.expires_at_ms { - return Err(grant.rule.label.clone()); - } - } let response = upstream.send(outbound); drop(materials); response @@ -376,6 +369,14 @@ async fn prepare_and_send( .map_err(|_| fail()) } +fn not_approved(label: &str, host: &str) -> Response { + let mut response = Response::new(full(Bytes::from(format!( + "secret {label} was not approved for {host}; the request was not sent" + )))); + *response.status_mut() = hyper::StatusCode::FORBIDDEN; + response +} + pub(super) fn body_buffer(_has_json: bool) -> Zeroizing> { Zeroizing::new(Vec::new()) } diff --git a/crates/aft/src/subc/consent.rs b/crates/aft/src/subc/consent.rs index 8ca2953ac1..ecfd871708 100644 --- a/crates/aft/src/subc/consent.rs +++ b/crates/aft/src/subc/consent.rs @@ -268,6 +268,7 @@ static TEST_TRANSPORT: LazyLock>>> = /// Install the production transport for this process's daemon connection. pub(super) fn install_subc_transport(connection_file: &Path) { + crate::secret::transport::install_connection_file(connection_file); let transport: Arc = Arc::new(SharedSubcConsent(Arc::new( SubcConsentTransport::new(connection_file.to_path_buf()), ))); diff --git a/crates/aft/src/subc/mod.rs b/crates/aft/src/subc/mod.rs index aca41bbf27..52b7a6dae7 100644 --- a/crates/aft/src/subc/mod.rs +++ b/crates/aft/src/subc/mod.rs @@ -245,7 +245,8 @@ pub(crate) use secret_consent::with_secret_bound_identity_for_test; #[cfg(test)] pub(crate) use secret_consent::with_secret_identity_for_test; pub(crate) use secret_consent::{ - current_call_scopes, current_secret_audit_scope, resolve_current_secret_label, LaunchDecision, + current_call_scopes, current_secret_audit_scope, current_secret_consent_context, + use_current_secret_label, LaunchDecision, }; mod standalone; #[doc(hidden)] diff --git a/crates/aft/src/subc/secret_consent.rs b/crates/aft/src/subc/secret_consent.rs index 79581bd0aa..4016576787 100644 --- a/crates/aft/src/subc/secret_consent.rs +++ b/crates/aft/src/subc/secret_consent.rs @@ -30,7 +30,7 @@ use crate::secret::eligibility::{alfonso_of_kind, COMMAND_SCOPE_KINDS}; use crate::secret::withhold_consent; use crate::secret::{ file_consent, CallScopes, ConsentOutcome, EligibilityForm, Ineligibility, LabelOutcome, - Resolution, SecretConsentRequest, SecretScope, + SecretConsentRequest, SecretScope, }; use std::sync::Arc; use std::time::Duration; @@ -72,16 +72,29 @@ fn decode_secret_elicitation(result: &serde_json::Value) -> Option<&str> { .filter(|id| id.starts_with("el_") && id.len() > 3) } -fn decode_secret_record(record: &serde_json::Value) -> ConsentOutcome { +fn decode_secret_record(record: &serde_json::Value, request_id: &str) -> ConsentOutcome { if record.get("kind").and_then(serde_json::Value::as_str) != Some("secret_use") { return consent_unavailable("Cingulate did not return a secret-use card"); } match record.get("state").and_then(serde_json::Value::as_str) { Some("pending") => ConsentOutcome::Pending, - // The answer is only a cue to retry the vault, never a grant or denial - // to apply locally. In particular, a decline also retries secret.use. - Some("answered") => ConsentOutcome::Answered, - Some("expired") => consent_unavailable("the secret-use card ended without an answer"), + Some("answered") => { + let answer = &record["secret_use_answer"]; + let decision = if let Some(items) = answer["items"].as_array() { + items + .iter() + .find(|item| item["request_id"].as_str() == Some(request_id)) + .and_then(|item| item["decision"].as_str()) + } else { + answer["decision"].as_str() + }; + if decision == Some("allow") { + ConsentOutcome::Answered + } else { + ConsentOutcome::Declined + } + } + Some("expired" | "withdrawn") => ConsentOutcome::Expired, _ => consent_unavailable("Cingulate returned an unknown secret-use card state"), } } @@ -114,7 +127,7 @@ impl SubcSecretConsent { })? } - async fn file(&self, request: &SecretConsentRequest) -> ConsentOutcome { + async fn file(&self, request: &SecretConsentRequest, hold: Duration) -> ConsentOutcome { // Cingulate deduplicates the exact ordered items for this filer. Refile // unchanged to recover the same card across retries or a lost reply, // without refreshing its deadline or introducing a local idempotency key. @@ -133,7 +146,6 @@ impl SubcSecretConsent { let Some(id) = decode_secret_elicitation(&filed) else { return consent_unavailable("Cingulate did not return a secret-use card id"); }; - let hold = super::consent::hold_budget(None); match self .call( request, @@ -143,13 +155,13 @@ impl SubcSecretConsent { ) .await { - Ok(record) => decode_secret_record(&record), + Ok(record) => decode_secret_record(&record, &request.request_id), Err(_) => consent_unavailable("Cingulate could not settle the secret-use card"), } } - fn dispatch(&self, request: &SecretConsentRequest) -> ConsentOutcome { - let future = self.file(request); + fn dispatch(&self, request: &SecretConsentRequest, hold: Duration) -> ConsentOutcome { + let future = self.file(request, hold); match tokio::runtime::Handle::try_current() { Ok(current) if current.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => @@ -164,7 +176,11 @@ impl SubcSecretConsent { impl crate::secret::SecretConsentTransport for SubcSecretConsent { fn file(&self, request: &SecretConsentRequest) -> ConsentOutcome { - self.dispatch(request) + self.dispatch(request, super::consent::hold_budget(None)) + } + + fn hold(&self, request: &SecretConsentRequest) -> ConsentOutcome { + self.dispatch(request, Duration::from_secs(600)) } fn withhold(&self, _request: &SecretConsentRequest) -> ConsentOutcome { @@ -327,7 +343,9 @@ pub(super) fn file_secret_consent( }; match file_consent(&request) { // No usable card or answer: nothing is held, nothing starts. - ConsentOutcome::Unavailable { .. } => Err(crate::secret::SECRET_ACCESS_UNAVAILABLE), + ConsentOutcome::Unavailable { .. } | ConsentOutcome::Declined | ConsentOutcome::Expired => { + Err(crate::secret::SECRET_ACCESS_UNAVAILABLE) + } ConsentOutcome::Answered => Ok(SecretConsentFiling::Answered), ConsentOutcome::Pending => Ok(SecretConsentFiling::Pending(PendingSecretConsent { request, @@ -346,37 +364,21 @@ pub(super) fn withdraw_secret_consent(pending: &PendingSecretConsent) { let _ = withhold_consent(&pending.request); } -/// What one launch decided for one label, before anything starts. -#[derive(Clone, Debug, PartialEq, Eq)] +/// Result of the fetch's addressed vault call, before sending upstream. +#[derive(Debug)] pub(crate) enum LaunchDecision { - /// The label may be used: the rule to follow and the grant to ask for - /// material with. - Proceed { - /// The label's rule, as the vault resolved it. - rule: secret_rules::location::LabelRule, - /// The grant the material call names. - grant_id: String, - expires_at_ms: u64, - }, - /// The launch is held until the user answers the filed ask. + Proceed(crate::secret::UseAnswer), Pending(PendingSecretConsent), - /// The launch ends with this outcome, whose code names the reason. Refuse(LabelOutcome), } -/// Resolve one scanned label for the launch on `identity`: the one entry a -/// bash or fetch launch calls for each label it found in the command text. -/// -/// The order is the outcome table's: an ineligible scope refuses with -/// `secret_refused` before the vault is asked anything, then resolution decides -/// between proceeding, holding the call for consent, and refusing. A label the -/// vault needs an answer for files one consent ask under the call's -/// authorization scope. An answer retries the vault immediately; a pending -/// card holds the launch until the caller retries. Neither answer grants here. -pub(crate) fn resolve_label_for_launch( +/// Fetch tools already know the target. Check eligibility before asking once, +/// then use the existing held-call consent path if approval is required. +pub(crate) fn use_label_for_fetch( identity: &RouteIdentity, vault: &dyn crate::secret::SecretVault, label: &str, + target: &crate::secret::ApiKeyTarget, ) -> LaunchDecision { let (scopes, _) = match call_scopes(identity) { Ok(scopes) => scopes, @@ -384,8 +386,6 @@ pub(crate) fn resolve_label_for_launch( return LaunchDecision::Refuse(crate::secret::refuse_ineligible(label, reason)) } }; - // Some vault adapters transiently receive material even at resolution. - // Harden here as well as at proxy startup so fetch and bash share the guard. #[cfg(unix)] if vault.can_release_material() && crate::secret_proxy::protect_secret_process().is_err() { return LaunchDecision::Refuse(LabelOutcome::Refuse { @@ -393,41 +393,23 @@ pub(crate) fn resolve_label_for_launch( message: crate::bash_background::secrets::PROXY_UNAVAILABLE.to_owned(), }); } - let resolution = vault.resolve(&scopes.authorization, label); - let outcome = crate::secret::outcome_for(label, &resolution); - match outcome { - LabelOutcome::Proceed => match resolution { - Resolution::Granted { - rule, - grant_id, - expires_at_ms, - } => LaunchDecision::Proceed { - rule, - grant_id, - expires_at_ms, - }, - // A Proceed outcome must correspond to the captured grant. - other => LaunchDecision::Refuse(crate::secret::outcome_for(label, &other)), - }, - LabelOutcome::FileConsent { - request_id, - statement_b64url, - } => match file_secret_consent(identity, &scopes, label, &request_id, &statement_b64url) { + let answer = vault.use_secret(&scopes.authorization, label, target); + if let crate::secret::UseAnswer::ApprovalRequired(pending) = answer { + return match file_secret_consent( + identity, + &scopes, + label, + &pending.request_id, + &pending.statement_b64url, + ) { Ok(SecretConsentFiling::Pending(pending)) => LaunchDecision::Pending(pending), Ok(SecretConsentFiling::Answered) => { - match vault.resolve(&scopes.authorization, label) { - Resolution::Granted { - rule, - grant_id, - expires_at_ms, - } => LaunchDecision::Proceed { - rule, - grant_id, - expires_at_ms, - }, - other => LaunchDecision::Refuse(crate::secret::outcome_for_second_resolve( - label, &other, - )), + let answer = vault.use_secret(&scopes.authorization, label, target); + let outcome = crate::secret::outcome_for_second_use(label, &answer); + if outcome.proceeds() { + LaunchDecision::Proceed(answer) + } else { + LaunchDecision::Refuse(outcome) } } Err(code) => LaunchDecision::Refuse(crate::secret::refuse( @@ -435,8 +417,13 @@ pub(crate) fn resolve_label_for_launch( label, "the consent ask could not be filed", )), - }, - refusal => LaunchDecision::Refuse(refusal), + }; + } + let outcome = crate::secret::outcome_for(label, &answer); + if outcome.proceeds() { + LaunchDecision::Proceed(answer) + } else { + LaunchDecision::Refuse(outcome) } } @@ -459,13 +446,14 @@ thread_local! { const { std::cell::RefCell::new(None) }; } -/// Resolve only with the identity installed by dispatch, never tool arguments. -pub(crate) fn resolve_current_secret_label( +/// Use only the dispatcher-installed identity, never tool arguments. +pub(crate) fn use_current_secret_label( vault: &dyn crate::secret::SecretVault, label: &str, + target: &crate::secret::ApiKeyTarget, ) -> LaunchDecision { CURRENT_IDENTITY.with(|slot| match slot.borrow().as_ref() { - Some(identity) => resolve_label_for_launch(identity, vault, label), + Some(identity) => use_label_for_fetch(identity, vault, label, target), None => LaunchDecision::Refuse(crate::secret::refuse_ineligible( label, Ineligibility::NoScope, @@ -473,6 +461,10 @@ pub(crate) fn resolve_current_secret_label( }) } +pub(crate) fn current_secret_consent_context() -> Option<(std::path::PathBuf, String)> { + current_identity().map(|identity| (identity.project_root.clone(), identity.session.clone())) +} + pub(super) fn current_identity() -> Option { CURRENT_IDENTITY.with(|slot| slot.borrow().clone()) } diff --git a/crates/aft/src/url_fetch.rs b/crates/aft/src/url_fetch.rs index dd237ea66c..8f58613001 100644 --- a/crates/aft/src/url_fetch.rs +++ b/crates/aft/src/url_fetch.rs @@ -818,15 +818,13 @@ pub(crate) fn create_secret_fetch_dir(storage_dir: &Path) -> Result { - /// The vault `material` and `audit` go to. + /// The vault each addressed request and audit record go to. pub vault: &'a dyn crate::secret::SecretVault, /// The call's authorization scope; `None` only when the URL holds no /// well-formed placeholder, in which case nothing is resolved or audited. pub authorization: Option, - /// Every label found in the URL, in order. Each has a rule in `rules`. + /// Every label found in the URL, in order. pub labels: Vec, - /// The rule the vault granted for each label, carrying its grant id. - pub rules: Vec, /// The fetch call's id: audit records name it, plus the hop index. pub call_id: String, } @@ -1430,30 +1428,56 @@ fn secret_hop( secret_fetch_refused(reason, logical) }; - let rules: Vec = secrets - .rules - .iter() - .filter(|rule| matches!(rule.location, Location::Query(_))) - .cloned() - .collect(); - let request = RequestTarget { + let mut rules = Vec::new(); + let mut materials = Vec::new(); + let present = secret_labels_in_url(logical).unwrap_or_default(); + let normalized = normalize_request_target(path).map_err(|_| refused(held, "malformed_path"))?; + let addressed = crate::secret::ApiKeyTarget { + scheme: "https".into(), host: authority.host.clone(), - port: authority.port, - scheme: "https", - target: &target, + port: Some(443), + path: String::from_utf8(normalized.wire).map_err(|_| refused(held, "malformed_path"))?, }; - let matched: Vec = rules - .iter() - .map(|rule| { - request - .matches(&RuleTarget { - host: &rule.host, - port: rule.port, - path_prefix: &rule.path_prefix, - }) - .is_ok() - }) - .collect(); + for label in &secrets.labels { + if authority.port != 443 || !present.contains(label) { + continue; + } + let answer = match crate::subc::use_current_secret_label(secrets.vault, label, &addressed) { + crate::subc::LaunchDecision::Proceed(answer) => answer, + crate::subc::LaunchDecision::Pending(pending) => { + return Err(UrlFetchError::secret( + crate::secret::SECRET_ACCESS_PENDING, + pending.message, + )) + } + crate::subc::LaunchDecision::Refuse(crate::secret::LabelOutcome::Refuse { + code, + message, + }) => return Err(UrlFetchError::secret(code, message)), + _ => return Err(refused(held, "missing_material")), + }; + if let crate::secret::UseAnswer::Released(released) = answer { + let (location, material) = released.into_parts(); + let location = secret_rules::location::parse_vault_location(&location) + .map_err(|_| refused(held, "missing_material"))?; + if !matches!(location, Location::Query(_)) { + continue; + } + if check_admissible(material.expose(), label, &location).is_err() { + return Err(refused(held, "material_not_admissible")); + } + held.hold(label, material.expose()) + .map_err(|_| refused(held, "replacement_discloses_key"))?; + rules.push(LabelRule { + label: label.clone(), + host: addressed.host.clone(), + port: 443, + path_prefix: addressed.path.clone(), + location, + }); + materials.push(Some(material)); + } + } // Plan with a stand-in key first: the rules decide where a key would go, // and refuse an ambiguous request, before the vault releases anything. let planning: Vec<(String, Vec)> = rules @@ -1466,45 +1490,6 @@ fn secret_hop( return Err(refused(&*held, refusal.reason())); } - // Material is asked only for a rule this hop actually substitutes. A - // rule whose host and path match while its placeholder is absent, not the - // whole value, or in another parameter needs no key, and asking anyway - // would spend a single-use grant on a request that sends none. - let mut materials = Vec::with_capacity(rules.len()); - for ((rule, matched), planned) in rules.iter().zip(&matched).zip(&plan.labels) { - if !matched || !planned.is_substituted() { - materials.push(None); - continue; - } - let Some(scope) = secrets.authorization.as_ref() else { - return Err(refused(&*held, "missing_material")); - }; - let material = match secrets.vault.material(scope, &rule.label, &rule.grant_id) { - Ok(material) => material, - Err(error) => { - let _ = emit_hop_audit(secrets, held, hop, Some(&authority), |_| { - (Decision::Refused, AuditReason::Refused) - }); - return Err( - match crate::secret::outcome_for_material(&rule.label, &error) { - crate::secret::LabelOutcome::Refuse { code, message } => { - UrlFetchError::secret(code, message) - } - _ => secret_fetch_refused("missing_material", logical), - }, - ); - } - }; - if check_admissible(material.expose(), &rule.label, &rule.location).is_err() { - return Err(refused(&*held, "material_not_admissible")); - } - // The matcher is installed before any byte of this hop is sent. - if held.hold(&rule.label, material.expose()).is_err() { - return Err(refused(&*held, "replacement_discloses_key")); - } - materials.push(Some(material)); - } - let query_start = path.len() + 1; let mut edits: Vec<(usize, usize, Zeroizing>)> = Vec::new(); for ((rule, outcome), material) in rules.iter().zip(&plan.labels).zip(&materials) { @@ -1600,14 +1585,9 @@ fn emit_hop_audit( .as_ref() .map(crate::secret::SecretScope::to_scope_ref), ); - let rule = secrets.rules.iter().find(|rule| &rule.label == label); - if let (Some(authority), Some(rule)) = (authority, rule) { - record = record.with_rule( - &authority.host, - authority.port, - &rule.path_prefix, - location_kind_token(&rule.location), - ); + if let Some(authority) = authority { + record.host = Some(authority.host.clone()); + record.port = Some(authority.port); } if record.to_json_line_with_keys(&keys).is_err() || secrets.vault.audit(&record).is_err() { failed = true; diff --git a/crates/secret-rules/src/location.rs b/crates/secret-rules/src/location.rs index 8c1882a696..648216823d 100644 --- a/crates/secret-rules/src/location.rs +++ b/crates/secret-rules/src/location.rs @@ -71,7 +71,7 @@ pub fn location_kind_token(location: &Location) -> String { } } -/// A label's rule, as resolved by the vault (V1). +/// The location and concrete target authorized for one request. #[derive(Debug, Clone, PartialEq, Eq)] pub struct LabelRule { pub label: String, @@ -79,9 +79,6 @@ pub struct LabelRule { pub port: u16, pub path_prefix: String, pub location: Location, - /// Opaque grant identifier; the vault compares it, the crate never parses - /// it. - pub grant_id: String, } /// Why a request is refused (P2, P4, and the target rules it depends on). From ebc9744bdec93759d78e97bcd0c5214279df3c0f Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:26:09 +0100 Subject: [PATCH 037/163] mason: migrate vault fixtures and vendor Cingulate decisions Vendor requester samples from Cingulate 56fa35f2 unchanged. Decode per-item allow/decline, expiry and withdrawn states. Replace grant-expiry assertions with the approved per-request contract: a release covers its addressed request; the next request must ask again. Uniform not_found now forwards literals, while transport failures still refuse. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit d33a0a87bb86729af552c9312164458ef748e23c) (cherry picked from commit 5f02ff270c741e593499cb602d8d8a4dd1e8b18d) --- crates/aft/src/bash_background/registry.rs | 14 +- .../aft/src/bash_background/secrets_tests.rs | 40 ++-- crates/aft/src/commands/url_resolution.rs | 47 ++-- crates/aft/src/secret/tests.rs | 123 +++++------ crates/aft/src/secret/transport_tests.rs | 13 +- crates/aft/src/secret/vault/fake.rs | 3 +- crates/aft/src/secret/wire_tests.rs | 50 +++-- crates/aft/src/secret_proxy/audit_tests.rs | 97 ++++----- crates/aft/src/secret_proxy/mod.rs | 1 - crates/aft/src/secret_proxy/request.rs | 2 +- crates/aft/src/secret_proxy/tests.rs | 88 ++++---- crates/aft/src/subc/bash.rs | 25 +-- crates/aft/src/subc/secret_consent_tests.rs | 202 +++++++++++++----- crates/aft/src/url_fetch.rs | 19 +- crates/secret-rules/tests/audit_location.rs | 2 - crates/secret-rules/tests/audit_r2.rs | 1 - crates/secret-rules/tests/location_rules.rs | 1 - crates/secret-rules/tests/target_rules.rs | 1 - crates/secret-rules/tests/vectors.rs | 1 - ...cingulate-secret-use-approval-v1.source.md | 6 + .../README.md | 40 ++++ .../SHA256SUMS | 12 ++ .../batch-phone-get.json | 1 + .../filer-await-allowed.json | 69 ++++++ .../filer-await-declined.json | 67 ++++++ .../filer-await-expired.json | 56 +++++ .../filer-await-partial.json | 77 +++++++ .../filer-await-pending.json | 55 +++++ .../filer-refusals.json | 92 ++++++++ .../filer-request-repeat.json | 19 ++ .../filer-request.json | 19 ++ .../secret-approval-v1.json | 111 ++++++++++ .../single-phone-get.json | 1 + 33 files changed, 1059 insertions(+), 296 deletions(-) create mode 100644 test-vectors/cingulate-secret-use-approval-v1.source.md create mode 100644 test-vectors/cingulate-secret-use-approval-v1/README.md create mode 100644 test-vectors/cingulate-secret-use-approval-v1/SHA256SUMS create mode 100644 test-vectors/cingulate-secret-use-approval-v1/batch-phone-get.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/filer-await-allowed.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/filer-await-declined.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/filer-await-expired.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/filer-await-partial.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/filer-await-pending.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/filer-refusals.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/filer-request-repeat.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/filer-request.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/secret-approval-v1.json create mode 100644 test-vectors/cingulate-secret-use-approval-v1/single-phone-get.json diff --git a/crates/aft/src/bash_background/registry.rs b/crates/aft/src/bash_background/registry.rs index b7b76b6078..ce38e6c0d6 100644 --- a/crates/aft/src/bash_background/registry.rs +++ b/crates/aft/src/bash_background/registry.rs @@ -3141,8 +3141,18 @@ impl BgTaskRegistry { append_remote_report(&mut snapshot.output_preview, &state.metadata); } #[cfg(unix)] - if let Some(runtime) = self.inner.secret_proxies.lock().get(&task.task_id).cloned() { - for line in runtime.lock().approval_status() { + if let Some(runtime) = self + .inner + .secret_proxies + .lock() + .ok() + .and_then(|proxies| proxies.get(&task.task_id).cloned()) + { + for line in runtime + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .approval_status() + { if !snapshot.output_preview.is_empty() { snapshot.output_preview.push('\n'); } diff --git a/crates/aft/src/bash_background/secrets_tests.rs b/crates/aft/src/bash_background/secrets_tests.rs index c261c3cc71..ad3341e409 100644 --- a/crates/aft/src/bash_background/secrets_tests.rs +++ b/crates/aft/src/bash_background/secrets_tests.rs @@ -1,7 +1,6 @@ use super::*; use crate::protocol::RawRequest; -use crate::secret::{Material, MaterialError, Resolution}; -use secret_rules::location::{LabelRule, Location}; +use crate::secret::{ApiKeyTarget, Material, ReleasedApiKey, UseAnswer}; use serde_json::json; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; @@ -36,7 +35,12 @@ impl Vault { const KEY: &[u8] = b"s4-test-api-key-0123456789"; impl SecretVault for Vault { - fn resolve(&self, scope: &crate::secret::SecretScope, label: &str) -> Resolution { + fn use_secret( + &self, + scope: &crate::secret::SecretScope, + label: &str, + target: &ApiKeyTarget, + ) -> UseAnswer { #[cfg(unix)] { assert!( @@ -64,25 +68,13 @@ impl SecretVault for Vault { .lock() .unwrap() .push(scope.kind().to_owned()); - Resolution::Granted { - rule: LabelRule { - label: label.into(), - host: "localhost".into(), - port: self.port, - path_prefix: "/api".into(), - location: Location::Bearer, - grant_id: "grant".into(), - }, - grant_id: "grant".into(), - expires_at_ms: u64::MAX, + if target.scheme != "https" + || target.host != "localhost" + || target.port != Some(443) + || target.path != "/api" + { + return UseAnswer::NotFound; } - } - fn material( - &self, - scope: &crate::secret::SecretScope, - label: &str, - grant_id: &str, - ) -> Result { self.materials.fetch_add(1, Ordering::SeqCst); assert_eq!(scope.reference(), "secret-bash"); assert_eq!(scope.owner(), "reserved:prefrontal-core"); @@ -91,12 +83,14 @@ impl SecretVault for Vault { self.expected_epoch.load(Ordering::SeqCst) ); assert_eq!(label, "test"); - assert_eq!(grant_id, "grant"); self.material_kinds .lock() .unwrap() .push(scope.kind().to_owned()); - Ok(Material::new(KEY.to_vec())) + UseAnswer::Released(ReleasedApiKey::new( + "authorization_bearer".into(), + Material::new(KEY.to_vec()), + )) } fn audit( &self, diff --git a/crates/aft/src/commands/url_resolution.rs b/crates/aft/src/commands/url_resolution.rs index ec82565488..d563978242 100644 --- a/crates/aft/src/commands/url_resolution.rs +++ b/crates/aft/src/commands/url_resolution.rs @@ -185,6 +185,12 @@ fn resolve_secret_fetch( vault: vault.as_ref(), authorization, labels: labels.to_vec(), + #[cfg(test)] + test_https_port: if options.allow_private { + url::Url::parse(file).ok().and_then(|url| url.port()) + } else { + None + }, call_id: req.id.clone(), }; let document = crate::url_fetch::fetch_url_with_secrets(file, &document_dir, options, &secrets) @@ -573,7 +579,7 @@ mod url_tests { mod secret_fetch { use super::super::*; - use crate::secret::{FakeVault, Resolution}; + use crate::secret::{FakeReply, FakeVault}; use secret_rules::location::{LabelRule, Location}; use std::io::{Read as _, Write as _}; use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener}; @@ -740,13 +746,12 @@ mod url_tests { port, path_prefix: prefix.into(), location: Location::Query("key".into()), - grant_id: String::new(), } } fn granted_vault(rule: LabelRule) -> FakeVault { let vault = FakeVault::new(); - vault.set_resolution("test", crate::secret::granted(rule, "grant-1")); + vault.set_answer("test", crate::secret::granted(rule)); vault.set_material("test", vec![KEY.to_vec()]); vault } @@ -962,7 +967,7 @@ mod url_tests { ); assert_documents_removed(&run, &cache); - assert_eq!(vault.resolve_calls(), 1); + assert_eq!(vault.use_calls(), 1); assert_eq!(vault.material_calls(), 1); let records = vault.audit_records(); assert_eq!(records.len(), 1, "{records:?}"); @@ -1240,7 +1245,7 @@ mod url_tests { "{command}" ); assert!(!run.text().contains("SENTINEL"), "{command}"); - assert_eq!(vault.resolve_calls(), 0, "{command}"); + assert_eq!(vault.use_calls(), 0, "{command}"); assert!(run.documents.is_empty()); } assert_eq!(cache_entries(&cache), before); @@ -1308,7 +1313,7 @@ mod url_tests { } #[test] - fn ineligible_or_refused_labels_fetch_nothing() { + fn ineligible_and_unavailable_fetch_nothing_but_not_found_is_literal() { let fixture = fixture(Default::default()); let server = serve(&fixture.tls, |_| Some(markdown(&echo_document()))); let port = server.addr.port(); @@ -1319,17 +1324,17 @@ mod url_tests { let vault = granted_vault(rule("api.secret.test", port, "/v1")); let unscoped = run(&fixture, &vault, &routes, &req, false); assert_eq!(unscoped.code(), Some(crate::secret::SECRET_REFUSED)); - assert_eq!(vault.resolve_calls(), 0); + assert_eq!(vault.use_calls(), 0); let denied = FakeVault::new(); - denied.set_resolution( - "test", - Resolution::Denied { - reason: "no".into(), - }, - ); + denied.set_answer("test", FakeReply::NotFound); let run_denied = run(&fixture, &denied, &routes, &req, true); - assert_eq!(run_denied.code(), Some(crate::secret::SECRET_REFUSED)); + assert!(run_denied.success); + assert_eq!(denied.material_calls(), 0); + assert_eq!( + denied.audit_summary(), + vec![("test".into(), secret_rules::audit::Decision::Literal)] + ); let unavailable = FakeVault::new(); let run_unavailable = run(&fixture, &unavailable, &routes, &req, true); @@ -1337,10 +1342,10 @@ mod url_tests { run_unavailable.code(), Some(crate::secret::SECRETS_UNAVAILABLE) ); - assert_eq!(unavailable.resolve_calls(), 1); + assert_eq!(unavailable.use_calls(), 1); - assert_eq!(server.connections.load(Ordering::SeqCst), 0); - for vault in [&vault, &denied, &unavailable] { + assert_eq!(server.connections.load(Ordering::SeqCst), 1); + for vault in [&vault, &unavailable] { let summary = vault.audit_summary(); assert_eq!( summary, @@ -1440,7 +1445,7 @@ mod url_tests { fn granted_vault_with(rule: LabelRule, key: &[u8]) -> FakeVault { let vault = FakeVault::new(); - vault.set_resolution("test", crate::secret::granted(rule, "grant-1")); + vault.set_answer("test", crate::secret::granted(rule)); vault.set_material("test", vec![key.to_vec()]); vault } @@ -2034,9 +2039,9 @@ mod url_tests { "https://api.secret.test:{port}/v1/doc.md?key={PLACEHOLDER}&other={{{{ck-secret:other}}}}" ); let vault = FakeVault::new(); - vault.set_resolution( + vault.set_answer( "test", - crate::secret::granted(rule("api.secret.test", port, "/v1"), "grant-1"), + crate::secret::granted(rule("api.secret.test", port, "/v1")), ); vault.set_material("test", vec![K_A.to_vec()]); let other = LabelRule { @@ -2044,7 +2049,7 @@ mod url_tests { location: Location::Query("other".into()), ..rule("api.secret.test", port, "/v1") }; - vault.set_resolution("other", crate::secret::granted(other, "grant-2")); + vault.set_answer("other", crate::secret::granted(other)); vault.set_material("other", vec![K_B.to_vec()]); let (run, documents) = run_with( &fixture, diff --git a/crates/aft/src/secret/tests.rs b/crates/aft/src/secret/tests.rs index 12e8412c08..f98979d77c 100644 --- a/crates/aft/src/secret/tests.rs +++ b/crates/aft/src/secret/tests.rs @@ -25,7 +25,6 @@ fn rule(label: &str) -> secret_rules::location::LabelRule { port: 443, path_prefix: "/v1".into(), location: Location::Bearer, - grant_id: "grant-1".into(), } } @@ -34,21 +33,27 @@ fn rule(label: &str) -> secret_rules::location::LabelRule { // --------------------------------------------------------------------------- #[test] -fn production_resolve_is_unavailable_for_every_label_and_scope() { - let vault = UnavailableVault; - let scope = production_scope(); +fn standalone_use_is_unavailable_for_every_label_and_scope() { for label in ["prod", "test", "repo"] { - assert_eq!(vault.resolve(&scope, label), Resolution::Unavailable); + assert!(matches!( + UnavailableVault.use_secret(&production_scope(), label, &target()), + UseAnswer::Unavailable + )); + } +} + +fn target() -> ApiKeyTarget { + ApiKeyTarget { + scheme: "https".into(), + host: "api.example.com".into(), + port: Some(443), + path: "/v1".into(), } - assert!(matches!( - vault.material(&scope, "prod", "grant-1"), - Err(MaterialError::Unavailable { .. }) - )); } #[test] fn an_unavailable_resolution_ends_the_launch_with_secrets_unavailable() { - let outcome = outcome_for("prod", &Resolution::Unavailable); + let outcome = outcome_for("prod", &UseAnswer::Unavailable); assert_eq!(outcome.code(), Some(SECRETS_UNAVAILABLE)); assert!(!outcome.proceeds()); // The message names the label it refused and never a key. @@ -63,15 +68,17 @@ fn an_unavailable_resolution_ends_the_launch_with_secrets_unavailable() { // --------------------------------------------------------------------------- #[test] -fn a_granted_resolution_proceeds() { - let outcome = outcome_for("repo", &granted(rule("repo"), "grant-1")); - assert_eq!(outcome, LabelOutcome::Proceed); - assert_eq!(outcome.code(), None); +fn a_released_request_proceeds() { + let answer = UseAnswer::Released(ReleasedApiKey::new( + "authorization_bearer".into(), + Material::new(b"test-secret".to_vec()), + )); + assert_eq!(outcome_for("repo", &answer), LabelOutcome::Proceed); } #[test] fn an_approval_required_resolution_files_a_consent_and_holds_the_call() { - let outcome = outcome_for("repo", &Resolution::ApprovalRequired(approval("req-1"))); + let outcome = outcome_for("repo", &UseAnswer::ApprovalRequired(approval("req-1"))); let LabelOutcome::FileConsent { request_id, statement_b64url, @@ -87,46 +94,22 @@ fn an_approval_required_resolution_files_a_consent_and_holds_the_call() { } #[test] -fn a_denied_resolution_refuses_naming_the_label() { - let outcome = outcome_for( - "repo", - &Resolution::Denied { - reason: "this device may not use repo".into(), - }, - ); - assert_eq!(outcome.code(), Some(SECRET_REFUSED)); - let LabelOutcome::Refuse { message, .. } = &outcome else { - panic!("a denial refuses the launch"); - }; - assert!(message.contains("repo"), "{message}"); - assert!( - message.contains("this device may not use repo"), - "{message}" +fn uniform_not_found_keeps_the_literal_request() { + assert_eq!( + outcome_for("repo", &UseAnswer::NotFound), + LabelOutcome::Proceed ); } #[test] -fn a_second_resolve_is_the_only_thing_that_can_grant() { - // Cingulate's answer alone never grants: a resolve that still asks again - // substitutes nothing, while a grant after the answer proceeds and a - // denial refuses. +fn a_phone_answer_never_grants_material_without_a_vault_release() { assert_eq!( - outcome_for_second_resolve("repo", &Resolution::ApprovalRequired(approval("req-1"))).code(), + outcome_for_second_use("repo", &UseAnswer::ApprovalRequired(approval("req-1"))).code(), Some(SECRETS_UNAVAILABLE) ); - assert!(outcome_for_second_resolve("repo", &granted(rule("repo"), "grant-1")).proceeds()); + assert!(outcome_for_second_use("repo", &UseAnswer::NotFound).proceeds()); assert_eq!( - outcome_for_second_resolve( - "repo", - &Resolution::Denied { - reason: "no".into() - } - ) - .code(), - Some(SECRET_REFUSED) - ); - assert_eq!( - outcome_for_second_resolve("repo", &Resolution::Unavailable).code(), + outcome_for_second_use("repo", &UseAnswer::Unavailable).code(), Some(SECRETS_UNAVAILABLE) ); } @@ -157,11 +140,14 @@ fn the_first_failing_label_decides_a_multi_label_launch() { } #[test] -fn a_material_lapse_refuses_without_filing_a_consent() { - let outcome = outcome_for_material("repo", &MaterialError::GrantLapsed); - assert_eq!(outcome.code(), Some(SECRET_REFUSED)); - // A lapse never asks for a new consent: the proxy is already live. - assert!(!matches!(outcome, LabelOutcome::FileConsent { .. })); +fn a_new_request_may_ask_for_a_fresh_approval() { + assert!(matches!( + outcome_for( + "repo", + &UseAnswer::ApprovalRequired(approval("next-request")) + ), + LabelOutcome::FileConsent { .. } + )); } // --------------------------------------------------------------------------- @@ -169,11 +155,9 @@ fn a_material_lapse_refuses_without_filing_a_consent() { // --------------------------------------------------------------------------- #[test] -fn the_fake_counts_resolve_and_material_calls_and_rotates_material() { +fn the_fake_counts_addressed_uses_and_rotates_material() { let mut script = FakeVaultScript::default(); - script - .resolutions - .insert("repo".into(), granted(rule("repo"), "grant-1")); + script.answers.insert("repo".into(), granted(rule("repo"))); script.material.insert( "repo".into(), vec![ @@ -183,22 +167,23 @@ fn the_fake_counts_resolve_and_material_calls_and_rotates_material() { ); let vault = FakeVault::with_script(script); let scope = production_scope(); + let UseAnswer::Released(first) = vault.use_secret(&scope, "repo", &target()) else { + panic!("release"); + }; + let UseAnswer::Released(second) = vault.use_secret(&scope, "repo", &target()) else { + panic!("release"); + }; + assert_eq!(first.material(), b"first-key-material-1"); + assert_eq!(second.material(), b"second-key-material"); + assert_eq!(vault.use_calls(), 2); + assert_eq!(vault.material_calls(), 2); + let mut other = target(); + other.host = "other.example.com".into(); assert!(matches!( - vault.resolve(&scope, "repo"), - Resolution::Granted { .. } + vault.use_secret(&scope, "repo", &other), + UseAnswer::NotFound )); - let first = vault.material(&scope, "repo", "grant-1").unwrap(); - let second = vault.material(&scope, "repo", "grant-1").unwrap(); - // One grant, two different values: the seam never assumes a grant returns - // the same bytes. - assert_eq!(first.expose(), b"first-key-material-1"); - assert_eq!(second.expose(), b"second-key-material"); - assert_eq!(vault.resolve_calls(), 1); assert_eq!(vault.material_calls(), 2); - assert_eq!( - vault.materials(), - vec!["repo".to_string(), "repo".to_string()] - ); } #[test] diff --git a/crates/aft/src/secret/transport_tests.rs b/crates/aft/src/secret/transport_tests.rs index c752d02d83..b304354013 100644 --- a/crates/aft/src/secret/transport_tests.rs +++ b/crates/aft/src/secret/transport_tests.rs @@ -197,8 +197,17 @@ fn secret_use_standalone_cannot_construct_a_module_transport_and_keeps_unavailab let stamp: ScopeStamp = serde_json::from_value(golden(0)["stamp"].clone()).unwrap(); let vault = crate::bash_background::secrets::vault(); assert!(matches!( - vault.resolve(&crate::secret::SecretScope::from_stamp(&stamp), "key"), - crate::secret::Resolution::Unavailable + vault.use_secret( + &crate::secret::SecretScope::from_stamp(&stamp), + "key", + &crate::secret::ApiKeyTarget { + scheme: "https".into(), + host: "x.test".into(), + port: Some(443), + path: "/repos/a".into() + } + ), + crate::secret::UseAnswer::Unavailable )); } diff --git a/crates/aft/src/secret/vault/fake.rs b/crates/aft/src/secret/vault/fake.rs index 2c1978ed28..5782c3f27e 100644 --- a/crates/aft/src/secret/vault/fake.rs +++ b/crates/aft/src/secret/vault/fake.rs @@ -234,7 +234,8 @@ impl SecretVault for FakeVault { } /// A matching record for an addressed test call. -pub fn granted(rule: LabelRule) -> FakeReply { +pub fn granted(mut rule: LabelRule) -> FakeReply { + rule.port = 443; FakeReply::Rule(rule) } diff --git a/crates/aft/src/secret/wire_tests.rs b/crates/aft/src/secret/wire_tests.rs index cd1ae2eeb2..ec4f7545bb 100644 --- a/crates/aft/src/secret/wire_tests.rs +++ b/crates/aft/src/secret/wire_tests.rs @@ -429,17 +429,21 @@ fn the_secret_use_client_sends_the_pinned_params_and_reads_a_release() { let transport = std::sync::Arc::new(Recording(std::sync::Mutex::new(Vec::new()))); let vault = SubcSecretVault::new(transport.clone()); let scope = SecretScope::from_stamp(&golden_stamp()); - let resolution = vault.resolve(&scope, "key"); - // A released key at resolution time is reported as a granted rule and the - // bytes themselves are dropped (zeroized), per the contract's material rule. - let Resolution::Granted { rule, .. } = resolution else { - panic!("a release at resolution time is a grant"); + let answer = vault.use_secret( + &scope, + "key", + &ApiKeyTarget { + scheme: "https".into(), + host: "x.test".into(), + port: Some(443), + path: "/repos/a".into(), + }, + ); + let UseAnswer::Released(released) = answer else { + panic!("a release keeps material for this request only"); }; - assert_eq!(rule.label, "key"); - assert!(matches!( - rule.location, - secret_rules::location::Location::Header(ref name) if name == "X-Api-Key" - )); + assert_eq!(released.location(), "header:X-Api-Key"); + assert_eq!(released.material(), b"sk-example"); let sent = transport.0.lock().unwrap(); assert_eq!(sent.len(), 1); assert_eq!(keys(&sent[0]), set(&["name", "stamp", "target"])); @@ -463,7 +467,16 @@ fn the_secret_use_client_reports_an_approval_as_pending() { } let vault = SubcSecretVault::new(std::sync::Arc::new(Pending)); let scope = SecretScope::from_stamp(&golden_stamp()); - let Resolution::ApprovalRequired(request) = vault.resolve(&scope, "key") else { + let UseAnswer::ApprovalRequired(request) = vault.use_secret( + &scope, + "key", + &ApiKeyTarget { + scheme: "https".into(), + host: "x.test".into(), + port: Some(443), + path: "/repos/a".into(), + }, + ) else { panic!("an approval_required answer is a pending resolution"); }; assert_eq!(request.request_id, "01HF7YAT000000000000000001"); @@ -471,7 +484,7 @@ fn the_secret_use_client_reports_an_approval_as_pending() { } #[test] -fn the_secret_use_client_maps_the_uniform_not_found_to_a_refusal() { +fn the_secret_use_client_preserves_uniform_not_found_for_literal_forwarding() { struct NotFound; impl SecretUseTransport for NotFound { fn secret_use(&self, _params: Value) -> Result { @@ -481,8 +494,17 @@ fn the_secret_use_client_maps_the_uniform_not_found_to_a_refusal() { let vault = SubcSecretVault::new(std::sync::Arc::new(NotFound)); let scope = SecretScope::from_stamp(&golden_stamp()); assert!(matches!( - vault.resolve(&scope, "key"), - Resolution::Denied { .. } + vault.use_secret( + &scope, + "key", + &ApiKeyTarget { + scheme: "https".into(), + host: "x.test".into(), + port: Some(443), + path: "/repos/a".into() + } + ), + UseAnswer::NotFound )); } diff --git a/crates/aft/src/secret_proxy/audit_tests.rs b/crates/aft/src/secret_proxy/audit_tests.rs index cb4d2572fd..90b74440ac 100644 --- a/crates/aft/src/secret_proxy/audit_tests.rs +++ b/crates/aft/src/secret_proxy/audit_tests.rs @@ -62,54 +62,55 @@ async fn audit_transfer_coding() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn audit_expiry_revocation_at_send() { - for revoke in [false, true] { - let server = Server::new("delay-h1").await; - let vault = Vault::new(); - let mut cfg = config( - vault.clone(), - server.address.port(), - &[("test", Location::Bearer)], - ); - let expiry = crate::secret::vault::now_ms() + 2000; - if !revoke { - cfg.grants[0].expires_at_ms = expiry; - } - let proxy = SecretProxy::start_with_tls(cfg, server.client_tls.clone()).unwrap(); - let mut client = Client::spawn(&proxy, &server); - proxy.arm_command(client.process.id()).unwrap(); - client.connect(false); - let worker = std::thread::spawn(move || { - let reply = client.request( - "/approved", - json!({"authorization":format!("Bearer {MARKER}")}), - "", - ); - (reply, client) - }); - tokio::time::timeout(Duration::from_secs(5), server.handshake_seen.notified()) - .await - .unwrap(); - if revoke { - vault.denied.store(true, Ordering::SeqCst); - } else { - tokio::time::sleep(Duration::from_millis( - expiry.saturating_sub(crate::secret::vault::now_ms()) + 20, - )) - .await; - } - server.tls_release.notify_one(); - let (reply, _client) = worker.join().unwrap(); - assert_eq!( - reply["status"], 403, - "authorization lapsed during upstream TLS: {reply}" - ); - assert!(server.seen.lock().is_empty()); - assert_eq!( - vault.calls.load(Ordering::SeqCst), - if revoke { 1 } else { 0 } +async fn audit_one_addressed_release_is_consumed_by_one_request() { + let server = Server::new("delay-h1").await; + let vault = Vault::new(); + let cfg = config( + vault.clone(), + server.address.port(), + &[("test", Location::Bearer)], + ); + let proxy = SecretProxy::start_with_tls(cfg, server.client_tls.clone()).unwrap(); + let mut client = Client::spawn(&proxy, &server); + proxy.arm_command(client.process.id()).unwrap(); + client.connect(false); + let worker = std::thread::spawn(move || { + let reply = client.request( + "/approved", + json!({"authorization":format!("Bearer {MARKER}")}), + "", ); - } + (reply, client) + }); + tokio::time::timeout(Duration::from_secs(5), server.handshake_seen.notified()) + .await + .unwrap(); + assert_eq!( + vault.calls.load(Ordering::SeqCst), + 1, + "ask before opening the upstream" + ); + vault.denied.store(true, Ordering::SeqCst); + server.tls_release.notify_one(); + let (reply, mut client) = worker.join().unwrap(); + assert_eq!( + reply["status"], 200, + "the release covers the request already admitted" + ); + assert_eq!( + client.request( + "/approved", + json!({"authorization":format!("Bearer {MARKER}")}), + "" + )["status"], + 403 + ); + assert_eq!(server.seen.lock().len(), 1); + assert_eq!( + vault.calls.load(Ordering::SeqCst), + 2, + "a later request must reach the vault again" + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -335,7 +336,7 @@ async fn audit_ipv6_literal() { server.address.port(), &[("test", Location::Bearer)], ); - cfg.grants[0].rule.host = "[::1]".into(); + vault.rules.lock().get_mut("test").unwrap().host = "[::1]".into(); let proxy = SecretProxy::start_with_tls(cfg, server.client_tls.clone()).unwrap(); let mut client = Client::spawn(&proxy, &server); proxy.arm_command(client.process.id()).unwrap(); diff --git a/crates/aft/src/secret_proxy/mod.rs b/crates/aft/src/secret_proxy/mod.rs index a4da0ce5e2..0ba47f17f6 100644 --- a/crates/aft/src/secret_proxy/mod.rs +++ b/crates/aft/src/secret_proxy/mod.rs @@ -75,7 +75,6 @@ use hyper::{Method, Request, Response, StatusCode}; use hyper_util::rt::{TokioExecutor, TokioIo}; use parking_lot::Mutex; use rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; -use secret_rules::location::Location; use secret_rules::scrub::{MatcherSet, Scrubber}; use secret_rules::target::ConnectAuthority; use tokio::net::{TcpListener, TcpStream}; diff --git a/crates/aft/src/secret_proxy/request.rs b/crates/aft/src/secret_proxy/request.rs index ad0851648d..56c8391ff8 100644 --- a/crates/aft/src/secret_proxy/request.rs +++ b/crates/aft/src/secret_proxy/request.rs @@ -205,7 +205,7 @@ async fn prepare_and_send( crate::secret::UseAnswer::Released(released) => { let (location, material) = released.into_parts(); let location = secret_rules::location::parse_vault_location(&location) - .map_err(|_| label.clone())?; + .ok_or_else(|| label.clone())?; check_admissible(material.expose(), label, &location).map_err(|_| label.clone())?; state .install(label, material.expose()) diff --git a/crates/aft/src/secret_proxy/tests.rs b/crates/aft/src/secret_proxy/tests.rs index 686d6b832b..838b1b256a 100644 --- a/crates/aft/src/secret_proxy/tests.rs +++ b/crates/aft/src/secret_proxy/tests.rs @@ -10,7 +10,9 @@ use secret_rules::location::Location; use serde_json::{json, Value}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use crate::secret::vault::{AuditError, Material, MaterialError, Resolution}; +use crate::secret::vault::{AuditError, Material, UseAnswer}; +use crate::secret::{ApiKeyTarget, ReleasedApiKey}; +use secret_rules::location::LabelRule; const K1: &[u8] = b"test-key-12345678"; const K2: &[u8] = b"rotated-key-987654321"; @@ -18,6 +20,7 @@ const MARKER: &str = "{{ck-secret:test}}"; const SPECIAL: &[u8] = b"alpha&percent%quote\"slash\\\r\nkey"; struct Vault { + rules: Mutex>, calls: AtomicUsize, denied: AtomicBool, rotate: AtomicBool, @@ -33,6 +36,7 @@ struct Vault { impl Vault { fn new() -> Arc { Arc::new(Self { + rules: Mutex::default(), calls: AtomicUsize::new(0), denied: AtomicBool::new(false), rotate: AtomicBool::new(false), @@ -48,16 +52,18 @@ impl Vault { } impl SecretVault for Vault { - fn resolve(&self, _: &SecretScope, _: &str) -> Resolution { - panic!("proxy must not resolve or file consent") - } - fn material( - &self, - scope: &SecretScope, - label: &str, - _: &str, - ) -> Result { + fn use_secret(&self, scope: &SecretScope, label: &str, target: &ApiKeyTarget) -> UseAnswer { assert_eq!(scope.reference(), "head"); + let Some(rule) = self.rules.lock().get(label).cloned() else { + return UseAnswer::NotFound; + }; + if rule.host != target.host + || !(target.path == rule.path_prefix + || target.path.starts_with(&format!("{}/", rule.path_prefix))) + { + return UseAnswer::NotFound; + } + let location = crate::secret::vault::fake::location_text(&rule.location); let count = self.calls.fetch_add(1, Ordering::SeqCst); if self.block_material.load(Ordering::Acquire) { self.material_entered.notify_one(); @@ -68,18 +74,24 @@ impl SecretVault for Vault { } if self.denied.load(Ordering::SeqCst) || self.refused_label.lock().as_deref() == Some(label) { - return Err(MaterialError::GrantLapsed); + return UseAnswer::Unavailable; } if self.special.load(Ordering::SeqCst) { - return Ok(Material::new(SPECIAL.to_vec())); + return UseAnswer::Released(ReleasedApiKey::new( + location, + Material::new(SPECIAL.to_vec()), + )); } - Ok(Material::new( - if self.rotate.load(Ordering::SeqCst) && count > 0 { - K2 - } else { - K1 - } - .to_vec(), + UseAnswer::Released(ReleasedApiKey::new( + location, + Material::new( + if self.rotate.load(Ordering::SeqCst) && count > 0 { + K2 + } else { + K1 + } + .to_vec(), + ), )) } fn audit(&self, record: &secret_rules::audit::AuditRecordV1) -> Result<(), AuditError> { @@ -99,25 +111,28 @@ fn scope(reference: &str) -> SecretScope { } fn config(vault: Arc, port: u16, locations: &[(&str, Location)]) -> ProxyConfig { - ProxyConfig { - vault, - authorization_scope: scope("head"), - lifetime_scope: scope("run"), - scanned_labels: locations.iter().map(|(l, _)| (*l).to_owned()).collect(), - grants: locations - .iter() - .map(|(label, location)| GrantedLabel { - rule: LabelRule { + *vault.rules.lock() = locations + .iter() + .map(|(label, location)| { + ( + (*label).to_owned(), + LabelRule { label: (*label).to_owned(), host: "localhost".into(), - port, + port: 443, path_prefix: "/approved".into(), location: location.clone(), - grant_id: "grant".into(), }, - expires_at_ms: u64::MAX, - }) - .collect(), + ) + }) + .collect(); + ProxyConfig { + vault, + authorization_scope: scope("head"), + lifetime_scope: scope("run"), + scanned_labels: locations.iter().map(|(l, _)| (*l).to_owned()).collect(), + consent_context: None, + test_https_port: Some(port), origin: "test-task".into(), } } @@ -1251,7 +1266,7 @@ async fn proxy_query_and_json_escape_material_without_partial_header_writes() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn proxy_expired_grant_refuses_without_material_and_tunnel_closes_on_stop() { +async fn proxy_unavailable_release_refuses_and_tunnel_closes_on_stop() { let server = Server::new("echo-request").await; let vault = Vault::new(); let mut cfg = config( @@ -1259,7 +1274,7 @@ async fn proxy_expired_grant_refuses_without_material_and_tunnel_closes_on_stop( server.address.port(), &[("test", Location::Bearer)], ); - cfg.grants[0].expires_at_ms = 0; + vault.denied.store(true, Ordering::SeqCst); let mut proxy = SecretProxy::start_with_tls(cfg, server.client_tls.clone()).unwrap(); let mut client = Client::spawn(&proxy, &server); proxy.arm_command(client.process.id()).unwrap(); @@ -1412,7 +1427,8 @@ fn proxy_only_accepts_grants_for_the_exact_launch_scan() { Some(ProxyError::InvalidRules) ); let mut cfg = make(); - cfg.grants[0].rule.location = Location::Header("Accept-Encoding".into()); + vault.rules.lock().get_mut("test").unwrap().location = + Location::Header("Accept-Encoding".into()); assert_eq!( SecretProxy::start_with_tls(cfg, tls).err(), Some(ProxyError::InvalidRules) diff --git a/crates/aft/src/subc/bash.rs b/crates/aft/src/subc/bash.rs index 6a1f58e396..2b68dc79b4 100644 --- a/crates/aft/src/subc/bash.rs +++ b/crates/aft/src/subc/bash.rs @@ -4880,21 +4880,13 @@ mod grant_path_tests { } fn secret_bash_scope_probe(req: RawRequest, _: &AppContext) -> Response { - use crate::secret::{Material, MaterialError, Resolution, SecretScope, SecretVault}; + use crate::secret::{ApiKeyTarget, SecretScope, SecretVault, UseAnswer}; struct Vault(std::sync::atomic::AtomicUsize); impl SecretVault for Vault { - fn resolve(&self, scope: &SecretScope, _: &str) -> Resolution { + fn use_secret(&self, scope: &SecretScope, _: &str, _: &ApiKeyTarget) -> UseAnswer { assert_eq!(scope.reference(), "secret-bash"); self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Resolution::Unavailable - } - fn material( - &self, - _: &SecretScope, - _: &str, - _: &str, - ) -> Result { - panic!("launch must not request material") + UseAnswer::Unavailable } fn audit( &self, @@ -4904,7 +4896,16 @@ mod grant_path_tests { } } let vault = Vault(std::sync::atomic::AtomicUsize::new(0)); - let decision = super::super::secret_consent::resolve_current_secret_label(&vault, "test"); + let decision = super::super::secret_consent::use_current_secret_label( + &vault, + "test", + &ApiKeyTarget { + scheme: "https".into(), + host: "x.test".into(), + port: Some(443), + path: "/repos/a".into(), + }, + ); let super::super::secret_consent::LaunchDecision::Refuse(outcome) = decision else { panic!("unavailable vault must refuse"); }; diff --git a/crates/aft/src/subc/secret_consent_tests.rs b/crates/aft/src/subc/secret_consent_tests.rs index 4da367bf12..9d6bd8327c 100644 --- a/crates/aft/src/subc/secret_consent_tests.rs +++ b/crates/aft/src/subc/secret_consent_tests.rs @@ -8,9 +8,9 @@ use super::*; use crate::secret::{ - first_failure, outcome_for, outcome_for_second_resolve, refuse_ineligible, ApprovalRequest, - FakeVault, FakeVaultScript, LabelOutcome, Resolution, SecretConsentTransport, SecretVault, - SECRETS_UNAVAILABLE, SECRET_REFUSED, + first_failure, outcome_for, outcome_for_second_use, refuse_ineligible, ApiKeyTarget, + ApprovalRequest, FakeReply, FakeVault, FakeVaultScript, LabelOutcome, SecretConsentTransport, + SecretVault, UseAnswer, SECRETS_UNAVAILABLE, SECRET_REFUSED, }; use serde_json::json; use std::sync::Mutex as StdMutex; @@ -375,14 +375,14 @@ fn every_ineligible_form_refuses_with_secret_refused_and_never_resolves() { // grant the label is installed, so a launch that reached it would // proceed: the counts below are only zero if the seam refused first. let vault = FakeVault::new(); - vault.set_resolution("repo", crate::secret::granted(rule(), "grant-1")); + vault.set_answer("repo", crate::secret::granted(rule())); vault.set_material("repo", vec![b"first-key-material-1".to_vec()]); - let decision = resolve_label_for_launch(&identity, &vault, "repo"); + let decision = use_label_for_fetch(&identity, &vault, "repo", &target()); let LaunchDecision::Refuse(outcome) = decision else { panic!("{name}: an ineligible scope must refuse, got {decision:?}"); }; assert_eq!(outcome.code(), Some(SECRET_REFUSED), "{name}"); - assert_eq!(vault.resolve_calls(), 0, "{name}: no resolve may happen"); + assert_eq!(vault.use_calls(), 0, "{name}: no resolve may happen"); assert_eq!( vault.material_calls(), 0, @@ -433,8 +433,8 @@ impl SecretConsentTransport for RecordingTransport { } } -fn approval(request_id: &str) -> Resolution { - Resolution::ApprovalRequired(ApprovalRequest { +fn approval(request_id: &str) -> FakeReply { + FakeReply::ApprovalRequired(ApprovalRequest { request_id: request_id.into(), statement_b64url: "c3RhdGVtZW50".into(), expires_at_ms: crate::secret::now_ms() + 60_000, @@ -444,11 +444,10 @@ fn approval(request_id: &str) -> Resolution { fn rule() -> secret_rules::location::LabelRule { secret_rules::location::LabelRule { label: "repo".into(), - host: "api.example.com".into(), + host: "x.test".into(), port: 443, - path_prefix: "/v1".into(), + path_prefix: "/repos/a".into(), location: secret_rules::location::Location::Bearer, - grant_id: "grant-1".into(), } } @@ -457,7 +456,7 @@ fn a_head_call_goes_pending_then_resolves_again_then_substitutes() { let head = head(); let (scopes, _) = call_scopes(&head).expect("a head call is eligible"); let mut script = FakeVaultScript::default(); - script.resolutions.insert("repo".into(), approval("req-1")); + script.answers.insert("repo".into(), approval("req-1")); script .material .insert("repo".into(), vec![b"first-key-material-1".to_vec()]); @@ -470,7 +469,10 @@ fn a_head_call_goes_pending_then_resolves_again_then_substitutes() { // First resolve: the vault is waiting, so the call is held before a child, // a job or a proxy exists. - let first = outcome_for("repo", &vault.resolve(&scopes.authorization, "repo")); + let first = outcome_for( + "repo", + &vault.use_secret(&scopes.authorization, "repo", &target()), + ); let LabelOutcome::FileConsent { request_id, statement_b64url, @@ -494,21 +496,29 @@ fn a_head_call_goes_pending_then_resolves_again_then_substitutes() { // Cingulate's answer alone grants nothing: the vault still refuses until it // grants, and only the second resolve decides. assert_eq!( - outcome_for_second_resolve("repo", &vault.resolve(&scopes.authorization, "repo")).code(), + outcome_for_second_use( + "repo", + &vault.use_secret(&scopes.authorization, "repo", &target()) + ) + .code(), Some(SECRETS_UNAVAILABLE) ); - vault.set_resolution("repo", crate::secret::granted(rule(), "grant-1")); - let second = outcome_for_second_resolve("repo", &vault.resolve(&scopes.authorization, "repo")); + vault.set_answer("repo", crate::secret::granted(rule())); + let second = outcome_for_second_use( + "repo", + &vault.use_secret(&scopes.authorization, "repo", &target()), + ); assert!(second.proceeds(), "{second:?}"); // The intercepted request takes its material once, under the lifetime // scope, from the grant the head's authorization allowed. - let material = vault - .material(&scopes.lifetime, "repo", "grant-1") - .expect("the grant hands out material"); - assert_eq!(material.expose(), b"first-key-material-1"); - assert_eq!(vault.resolve_calls(), 3); - assert_eq!(vault.material_calls(), 1); + let UseAnswer::Released(material) = vault.use_secret(&scopes.authorization, "repo", &target()) + else { + panic!("addressed release"); + }; + assert_eq!(material.material(), b"first-key-material-1"); + assert_eq!(vault.use_calls(), 4); + assert_eq!(vault.material_calls(), 2); } #[test] @@ -568,60 +578,63 @@ fn approval_reply() -> serde_json::Value { }}) } -// The saved Cingulate 0.12.0 secret-use-approval-v1.1 responses live in -// crates/aft/tests/fixtures/secret/cingulate-secret-use-v1.1/. These hashes pin -// the reviewed response bytes independently of the decoder. +// These requester records are vendored from Cingulate commit 56fa35f2. +// Independent hashes pin their exact bytes, not a decoder's reconstruction. const FILER_VECTORS: &[(&str, &str, &str)] = &[ ( "request", - include_str!("../../tests/fixtures/secret/cingulate-secret-use-v1.1/filer-request.json"), + include_str!( + "../../../../test-vectors/cingulate-secret-use-approval-v1/filer-request.json" + ), "be75724519b10547d1baacdee20095f753130251b1519c10342ab19ed42364c9", ), ( "request-repeat", include_str!( - "../../tests/fixtures/secret/cingulate-secret-use-v1.1/filer-request-repeat.json" + "../../../../test-vectors/cingulate-secret-use-approval-v1/filer-request-repeat.json" ), "be75724519b10547d1baacdee20095f753130251b1519c10342ab19ed42364c9", ), ( "pending", include_str!( - "../../tests/fixtures/secret/cingulate-secret-use-v1.1/filer-await-pending.json" + "../../../../test-vectors/cingulate-secret-use-approval-v1/filer-await-pending.json" ), "db39e1458276124efd66694d39f64e0ce34f0811c860b3ac4af2be6e6afbd0e6", ), ( "allowed", include_str!( - "../../tests/fixtures/secret/cingulate-secret-use-v1.1/filer-await-allowed.json" + "../../../../test-vectors/cingulate-secret-use-approval-v1/filer-await-allowed.json" ), "e5e665b918c4cb19750a8217457497c068eb4fb098df694da5f6dbd18511b286", ), ( "partial", include_str!( - "../../tests/fixtures/secret/cingulate-secret-use-v1.1/filer-await-partial.json" + "../../../../test-vectors/cingulate-secret-use-approval-v1/filer-await-partial.json" ), "8ebc4ce81712071e97c1a53b471722c1ac83020ed468723b83fcb674785a6b99", ), ( "declined", include_str!( - "../../tests/fixtures/secret/cingulate-secret-use-v1.1/filer-await-declined.json" + "../../../../test-vectors/cingulate-secret-use-approval-v1/filer-await-declined.json" ), "f49061b64579f5601883bd466aea4d0951915fc4e121c3e1390830003acc0c84", ), ( "expired", include_str!( - "../../tests/fixtures/secret/cingulate-secret-use-v1.1/filer-await-expired.json" + "../../../../test-vectors/cingulate-secret-use-approval-v1/filer-await-expired.json" ), "95eb5015d42f4e3d059729696f12ac0bfc2447a6f74b24c450bef36a39b0a6c4", ), ( "refusals", - include_str!("../../tests/fixtures/secret/cingulate-secret-use-v1.1/filer-refusals.json"), + include_str!( + "../../../../test-vectors/cingulate-secret-use-approval-v1/filer-refusals.json" + ), "66740bda26d96c0abd25e22244965209fcdee04e120bcd4d7214644e0580fa65", ), ]; @@ -662,21 +675,41 @@ fn secret_filer_vectors_are_pinned_and_decode_the_published_card_states() { decode_secret_elicitation(&filer_reply("request").unwrap()), Some("el_1") ); - for name in ["allowed", "partial", "declined"] { + for name in ["allowed", "partial"] { assert_eq!( - decode_secret_record(&filer_reply(name).unwrap()), + decode_secret_record(&filer_reply(name).unwrap(), "01HF7YAT000000000000000001"), ConsentOutcome::Answered, "{name}" ); } assert_eq!( - decode_secret_record(&filer_reply("pending").unwrap()), + decode_secret_record( + &filer_reply("pending").unwrap(), + "01HF7YAT000000000000000001" + ), ConsentOutcome::Pending ); assert!(matches!( - decode_secret_record(&filer_reply("expired").unwrap()), - ConsentOutcome::Unavailable { .. } + decode_secret_record( + &filer_reply("expired").unwrap(), + "01HF7YAT000000000000000001" + ), + ConsentOutcome::Expired )); + assert_eq!( + decode_secret_record( + &filer_reply("partial").unwrap(), + "01HF7YAT000000000000000002" + ), + ConsentOutcome::Declined + ); + assert_eq!( + decode_secret_record( + &filer_reply("declined").unwrap(), + "01HF7YAT000000000000000001" + ), + ConsentOutcome::Declined + ); } #[derive(Default)] @@ -770,8 +803,8 @@ fn a_refiled_secret_launch_recovers_the_same_card_and_exact_statement() { let vault = crate::secret::SubcSecretVault::new(wire.clone()); let identity = head(); for decision in [ - resolve_label_for_launch(&identity, &vault, "key"), - runtime.block_on(async { resolve_label_for_launch(&identity, &vault, "key") }), + use_label_for_fetch(&identity, &vault, "key", &target()), + runtime.block_on(async { use_label_for_fetch(&identity, &vault, "key", &target()) }), ] { assert!(matches!(decision, LaunchDecision::Pending(_))); } @@ -806,7 +839,7 @@ fn a_refiled_secret_launch_recovers_the_same_card_and_exact_statement() { } #[test] -fn secret_allow_partial_and_decline_retry_secret_use_and_only_the_vault_decides() { +fn secret_allow_and_allowed_partial_retry_but_decline_never_retries() { let runtime = secret_consent_runtime(); for state in ["allowed", "partial", "declined"] { for (reply, expected_code) in [ @@ -816,7 +849,7 @@ fn secret_allow_partial_and_decline_retry_secret_use_and_only_the_vault_decides( ), ( json!({"result": {"error": {"code": "not_found", "class": "permanent"}}}), - Some(SECRET_REFUSED), + None, ), (approval_reply(), Some(SECRETS_UNAVAILABLE)), ] { @@ -828,7 +861,16 @@ fn secret_allow_partial_and_decline_retry_secret_use_and_only_the_vault_decides( let wire = secret_use_script(vec![approval_reply(), reply]); let vault = crate::secret::SubcSecretVault::new(wire.clone()); let identity = head(); - let decision = resolve_label_for_launch(&identity, &vault, "key"); + let decision = use_label_for_fetch(&identity, &vault, "key", &target()); + if state == "declined" { + assert!(matches!(decision, LaunchDecision::Refuse(_))); + assert_eq!( + wire.calls.lock().unwrap().len(), + 1, + "a decline is final, without another vault call" + ); + continue; + } match expected_code { None => assert!( matches!(decision, LaunchDecision::Proceed { .. }), @@ -878,7 +920,8 @@ fn expired_or_refused_secret_cards_end_the_launch_without_retrying_the_vault() { let _guard = install_cards(&runtime, cards.clone()); let wire = secret_use_script(vec![approval_reply()]); let vault = crate::secret::SubcSecretVault::new(wire.clone()); - let LaunchDecision::Refuse(outcome) = resolve_label_for_launch(&head(), &vault, "key") + let LaunchDecision::Refuse(outcome) = + use_label_for_fetch(&head(), &vault, "key", &target()) else { panic!("an expired or refused card cannot start a launch") }; @@ -899,14 +942,13 @@ fn secret_cards_reject_unknown_or_permission_reply_shapes() { for record in [ json!({"kind": "secret_use", "state": "granted"}), json!({"kind": "secret_use", "state": "declined"}), - json!({"kind": "secret_use", "state": "withdrawn"}), json!({"kind": "secret_use", "state": "settled", "outcome": "allow_once"}), json!({"kind": "tool_permission", "state": "granted"}), json!({"kind": "secret_use", "state": "future"}), json!({"state": "granted"}), ] { assert!(matches!( - decode_secret_record(&record), + decode_secret_record(&record, "01HF7YAT000000000000000001"), ConsentOutcome::Unavailable { .. } )); } @@ -929,7 +971,8 @@ fn withholding_a_secret_card_fails_closed_without_an_unsupported_method() { let _guard = install_cards(&runtime, cards.clone()); let wire = secret_use_script(vec![approval_reply()]); let vault = crate::secret::SubcSecretVault::new(wire); - let LaunchDecision::Pending(pending) = resolve_label_for_launch(&head(), &vault, "key") else { + let LaunchDecision::Pending(pending) = use_label_for_fetch(&head(), &vault, "key", &target()) + else { panic!("the card is pending") }; assert!(matches!( @@ -975,11 +1018,14 @@ fn the_production_vault_ends_an_eligible_launch_with_secrets_unavailable() { let (scopes, _) = call_scopes(&head()).expect("a head call is eligible"); let vault = crate::secret::UnavailableVault; assert_eq!( - vault.resolve(&scopes.authorization, "repo"), - Resolution::Unavailable + matches!( + vault.use_secret(&scopes.authorization, "repo", &target()), + UseAnswer::Unavailable + ), + true ); assert_eq!( - crate::secret::outcome_for("repo", &Resolution::Unavailable).code(), + crate::secret::outcome_for("repo", &UseAnswer::Unavailable).code(), Some(SECRETS_UNAVAILABLE) ); } @@ -990,7 +1036,7 @@ fn the_production_vault_never_hardens_the_process() { // own /proc counters. A vault that can never release a key must not // trigger it, even when an eligible launch names a placeholder. let LaunchDecision::Refuse(outcome) = - resolve_label_for_launch(&head(), &crate::secret::UnavailableVault, "repo") + use_label_for_fetch(&head(), &crate::secret::UnavailableVault, "repo", &target()) else { panic!("the production vault refuses every launch") }; @@ -1122,12 +1168,60 @@ fn a_run_bound_to_an_unauthorized_head_stamp_is_ineligible() { assert_eq!(call_scopes(&run), Err(Ineligibility::HeadScopeNotAlfonso)); let vault = FakeVault::new(); - vault.set_resolution("repo", crate::secret::granted(rule(), "grant-1")); - let decision = resolve_label_for_launch(&run, &vault, "repo"); + vault.set_answer("repo", crate::secret::granted(rule())); + let decision = use_label_for_fetch(&run, &vault, "repo", &target()); let LaunchDecision::Refuse(outcome) = decision else { panic!("an unauthorized head stamp must refuse, got {decision:?}"); }; assert_eq!(outcome.code(), Some(SECRET_REFUSED)); - assert_eq!(vault.resolve_calls(), 0); + assert_eq!(vault.use_calls(), 0); assert_eq!(vault.material_calls(), 0); } + +fn target() -> ApiKeyTarget { + ApiKeyTarget { + scheme: "https".into(), + host: "x.test".into(), + port: Some(443), + path: "/repos/a".into(), + } +} + +#[test] +fn every_vendored_secret_approval_sample_matches_its_source_checksum_and_decodes() { + use sha2::{Digest, Sha256}; + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../test-vectors/cingulate-secret-use-approval-v1"); + let manifest = + include_str!("../../../../test-vectors/cingulate-secret-use-approval-v1/SHA256SUMS"); + let mut count = 0; + for line in manifest.lines() { + let (hash, name) = line.split_once(" ").unwrap(); + let bytes = std::fs::read(root.join(name)).unwrap(); + assert_eq!(format!("{:x}", Sha256::digest(&bytes)), hash, "{name}"); + if name.ends_with(".json") { + let sample: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert!(sample.is_object(), "{name}"); + if name.starts_with("filer-await-") { + let outcome = + decode_secret_record(&sample["reply"]["result"], "01HF7YAT000000000000000001"); + assert!( + !matches!(outcome, ConsentOutcome::Unavailable { .. }), + "{name}" + ); + } + } + count += 1; + } + assert_eq!(count, 12); + let fallback = json!({"kind":"secret_use", "state":"answered", "answered_choice_id":"allow", "secret_use_answer":{"decision":"allow"}}); + assert_eq!( + decode_secret_record(&fallback, "one"), + ConsentOutcome::Answered + ); + let withdrawn = json!({"kind":"secret_use", "state":"withdrawn"}); + assert_eq!( + decode_secret_record(&withdrawn, "one"), + ConsentOutcome::Expired + ); +} diff --git a/crates/aft/src/url_fetch.rs b/crates/aft/src/url_fetch.rs index 8f58613001..242b826978 100644 --- a/crates/aft/src/url_fetch.rs +++ b/crates/aft/src/url_fetch.rs @@ -22,12 +22,11 @@ use reqwest::header::{ use reqwest::redirect::Policy; use secret_rules::audit::{AuditReason, AuditRecordV1, Decision}; use secret_rules::location::{ - evaluate_request_with_connect, location_kind_token, LabelOutcome, LabelRule, Location, - RequestView, + evaluate_request_with_connect, LabelOutcome, LabelRule, Location, RequestView, }; use secret_rules::material::{check_admissible, encode_for_location}; use secret_rules::scrub::{MatcherSet, ScrubError, Scrubber}; -use secret_rules::target::{normalize_request_target, ConnectAuthority, RequestTarget, RuleTarget}; +use secret_rules::target::{normalize_request_target, ConnectAuthority}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use url::Url; @@ -825,6 +824,9 @@ pub struct SecretFetch<'a> { pub authorization: Option, /// Every label found in the URL, in order. pub labels: Vec, + /// Map an ephemeral test upstream to the vault's HTTPS/443 target. + #[cfg(test)] + pub test_https_port: Option, /// The fetch call's id: audit records name it, plus the hop index. pub call_id: String, } @@ -1439,7 +1441,12 @@ fn secret_hop( path: String::from_utf8(normalized.wire).map_err(|_| refused(held, "malformed_path"))?, }; for label in &secrets.labels { - if authority.port != 443 || !present.contains(label) { + let mut https_port = authority.port == 443; + #[cfg(test)] + { + https_port |= secrets.test_https_port == Some(authority.port); + } + if !https_port || !present.contains(label) { continue; } let answer = match crate::subc::use_current_secret_label(secrets.vault, label, &addressed) { @@ -1459,7 +1466,7 @@ fn secret_hop( if let crate::secret::UseAnswer::Released(released) = answer { let (location, material) = released.into_parts(); let location = secret_rules::location::parse_vault_location(&location) - .map_err(|_| refused(held, "missing_material"))?; + .ok_or_else(|| refused(held, "missing_material"))?; if !matches!(location, Location::Query(_)) { continue; } @@ -1471,7 +1478,7 @@ fn secret_hop( rules.push(LabelRule { label: label.clone(), host: addressed.host.clone(), - port: 443, + port: authority.port, path_prefix: addressed.path.clone(), location, }); diff --git a/crates/secret-rules/tests/audit_location.rs b/crates/secret-rules/tests/audit_location.rs index f38127bf7f..015ee8bad4 100644 --- a/crates/secret-rules/tests/audit_location.rs +++ b/crates/secret-rules/tests/audit_location.rs @@ -21,7 +21,6 @@ fn rule(label: &str, location: Location) -> LabelRule { port: 443, path_prefix: "/v1".to_string(), location, - grant_id: "grant".to_string(), } } @@ -94,7 +93,6 @@ fn redirect_authority_is_decided_per_label() { port: 443, path_prefix: "/v1".to_string(), location: Location::Query("key".to_string()), - grant_id: "grant".to_string(), }, ]; let hop = RequestTarget { diff --git a/crates/secret-rules/tests/audit_r2.rs b/crates/secret-rules/tests/audit_r2.rs index 4af324b604..d7c17b3c05 100644 --- a/crates/secret-rules/tests/audit_r2.rs +++ b/crates/secret-rules/tests/audit_r2.rs @@ -23,7 +23,6 @@ fn rule(label: &str, location: Location) -> LabelRule { port: 443, path_prefix: "/v1".to_string(), location, - grant_id: "grant".to_string(), } } diff --git a/crates/secret-rules/tests/location_rules.rs b/crates/secret-rules/tests/location_rules.rs index 5517bb5b6f..4ce2f7a01e 100644 --- a/crates/secret-rules/tests/location_rules.rs +++ b/crates/secret-rules/tests/location_rules.rs @@ -15,7 +15,6 @@ fn rule(location: Location) -> LabelRule { port: 443, path_prefix: "/v1".to_string(), location, - grant_id: "grant-1".to_string(), } } diff --git a/crates/secret-rules/tests/target_rules.rs b/crates/secret-rules/tests/target_rules.rs index 829af01253..582d5aa7a0 100644 --- a/crates/secret-rules/tests/target_rules.rs +++ b/crates/secret-rules/tests/target_rules.rs @@ -232,7 +232,6 @@ fn a_redirect_hop_to_another_host_scheme_or_port_is_not_authorized() { port: 443, path_prefix: "/v1".to_string(), location: Location::Query("key".to_string()), - grant_id: "g".to_string(), }; let rules = [rule]; let same = request("api.example.com", 443, "https", "/v1/next"); diff --git a/crates/secret-rules/tests/vectors.rs b/crates/secret-rules/tests/vectors.rs index 0d39933584..98e223478a 100644 --- a/crates/secret-rules/tests/vectors.rs +++ b/crates/secret-rules/tests/vectors.rs @@ -165,7 +165,6 @@ fn case_rule(value: &Value) -> Result { .map(support::string) .unwrap_or_else(|| "/v1".to_string()), location, - grant_id: "grant".to_string(), }) } diff --git a/test-vectors/cingulate-secret-use-approval-v1.source.md b/test-vectors/cingulate-secret-use-approval-v1.source.md new file mode 100644 index 0000000000..5804cbd9aa --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1.source.md @@ -0,0 +1,6 @@ +# Vendored Cingulate requester samples + +`cingulate-secret-use-approval-v1/` is an untouched copy of Cingulate's +`test-vectors/secret-use-approval-v1/` at source commit `56fa35f2`. +Its twelve files are pinned by the accompanying `SHA256SUMS`. The keys and +signatures are public deterministic test material, not live credentials. diff --git a/test-vectors/cingulate-secret-use-approval-v1/README.md b/test-vectors/cingulate-secret-use-approval-v1/README.md new file mode 100644 index 0000000000..04f13fa9ac --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/README.md @@ -0,0 +1,40 @@ +# Secret-use approvals v1 + +`secret-approval-v1.json` is copied byte for byte from Claustrum's +`secret-approval-vectors-v1` tag, commit `cefc5c7041ffe93d17c4fcc80e5847260f3ad121`, +at `crates/credentials-core/tests/vectors/secret-approval-v1.json`. +Its SHA-256 is `efd4ba60641586e143bce81087e4f9f5aaf8a99ffede74190303da7d4495cc27`. +These are public deterministic test keys, never production credentials. + +`single-phone-get.json` and `batch-phone-get.json` are untouched response bytes +from Cingulate's real `consent.get` path on a stamped phone route advertising +`clientFeatures: ["secret-use-approval"]`. They file the `approval` and +`batch-omitted` statements respectively, at 1700000000000 ms, in fresh scratch +stores. Phone records use `secretUse`; its nested vault items retain snake_case. +The phone renders requested access from the statements, not the generic title. + +Filer samples are framed `consent.request` and `consent.await` calls filed as +`reserved:aft` with the vault's `approval` statement (`batch-omitted` for the +partial card), at 1700000000000 ms in fresh scratch stores. Each file holds the +exact method, request params and handler reply envelope, so AFT and Cerebellum +can build decoders against real replies. Filer records use `snake_case` with +`secret_use` and `secret_use_answer`; refusals carry `code` and `message`. + +| File | What it shows | +|---|---| +| `secret-approval-v1.json` | Vault goldens copied byte for byte, never production credentials. | +| `single-phone-get.json` | Phone `consent.get` for the `approval` statement. | +| `batch-phone-get.json` | Phone `consent.get` for the `batch-omitted` statements. | +| `filer-request.json` | Filer `consent.request` reply with the new `elicitation_id`. | +| `filer-request-repeat.json` | Same filing repeated, returning the same `elicitation_id`. | +| `filer-await-pending.json` | Filer `consent.await` with zero timeout before any answer, pending. | +| `filer-await-allowed.json` | Filer `consent.await` after an allow envelope, with `secret_use_answer`. | +| `filer-await-partial.json` | Filer `consent.await` for the batch card after its one-item envelope. | +| `filer-await-declined.json` | Filer `consent.await` after a decline, every item declined. | +| `filer-await-expired.json` | Filer `consent.await` after the earliest expiry passes on the test clock. | +| `filer-refusals.json` | Four `consent.request` refusals: agent mismatch, non-canonical, id reuse, broca. | + +Regenerate with `cargo run -p cingulate-vectors --example generate_secret_use_vectors --locked`. +Use `--check` to compare without writing. The `secret_use` integration tests +regenerate and compare both served records byte for byte, exercise the three +use-approval cases through filing/answering, and verify checksum coverage. diff --git a/test-vectors/cingulate-secret-use-approval-v1/SHA256SUMS b/test-vectors/cingulate-secret-use-approval-v1/SHA256SUMS new file mode 100644 index 0000000000..71bbad77ea --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/SHA256SUMS @@ -0,0 +1,12 @@ +1e3997a231366c5892f2906cbd2b269664e68944edda60c4c1442b958ebe7058 README.md +7c940c153cb0a6ab8e3f8062d6c027a95c4dc3368b19dd5a72629766714b3f59 batch-phone-get.json +e5e665b918c4cb19750a8217457497c068eb4fb098df694da5f6dbd18511b286 filer-await-allowed.json +f49061b64579f5601883bd466aea4d0951915fc4e121c3e1390830003acc0c84 filer-await-declined.json +95eb5015d42f4e3d059729696f12ac0bfc2447a6f74b24c450bef36a39b0a6c4 filer-await-expired.json +8ebc4ce81712071e97c1a53b471722c1ac83020ed468723b83fcb674785a6b99 filer-await-partial.json +db39e1458276124efd66694d39f64e0ce34f0811c860b3ac4af2be6e6afbd0e6 filer-await-pending.json +66740bda26d96c0abd25e22244965209fcdee04e120bcd4d7214644e0580fa65 filer-refusals.json +be75724519b10547d1baacdee20095f753130251b1519c10342ab19ed42364c9 filer-request-repeat.json +be75724519b10547d1baacdee20095f753130251b1519c10342ab19ed42364c9 filer-request.json +efd4ba60641586e143bce81087e4f9f5aaf8a99ffede74190303da7d4495cc27 secret-approval-v1.json +81794c4c98c77fb57813d8cb4c396c52da17bab4c1e8e7f56c113f4a521cb4a8 single-phone-get.json diff --git a/test-vectors/cingulate-secret-use-approval-v1/batch-phone-get.json b/test-vectors/cingulate-secret-use-approval-v1/batch-phone-get.json new file mode 100644 index 0000000000..bb49c19bbd --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/batch-phone-get.json @@ -0,0 +1 @@ +{"result":{"createdAtMs":1700000000000,"default":"decline","elicitationId":"el_1","expiresAtMs":1700000600000,"facts":[],"kind":"secret_use","lateExecution":"notify_only","materialDamage":false,"onExpiry":"deny","options":[{"effect":"grant","id":"allow","label":"Allow"},{"effect":"decline","id":"decline","label":"Decline"}],"prompt":"Review the vault's requested access.","requesterLabel":"reserved:aft","requesterName":"AFT","scope":{},"secretUse":{"items":[{"request_id":"01HF7YAT000000000000000001","statement_b64url":"eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0"},{"request_id":"01HF7YAT000000000000000002","statement_b64url":"eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMiIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0"}]},"state":"pending","subject":{"agentId":"agent-A","headAgentId":"agent-A","label":"agent-A"},"target":{"kind":"secret_use","label":"Vault access"},"title":"Secret use approval","urgency":"normal"}} \ No newline at end of file diff --git a/test-vectors/cingulate-secret-use-approval-v1/filer-await-allowed.json b/test-vectors/cingulate-secret-use-approval-v1/filer-await-allowed.json new file mode 100644 index 0000000000..0db1c739e3 --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/filer-await-allowed.json @@ -0,0 +1,69 @@ +{ + "method": "consent.await", + "params": { + "elicitation_id": "el_1", + "timeout_ms": 0 + }, + "reply": { + "result": { + "answered_at_ms": 1700000000000, + "answered_choice_id": "allow", + "answered_device": "11111111111111111111111111111111", + "created_at_ms": 1700000000000, + "default": "decline", + "elicitation_id": "el_1", + "expires_at_ms": 1700000600000, + "facts": [], + "kind": "secret_use", + "late_execution": "notify_only", + "material_damage": false, + "on_expiry": "deny", + "options": [ + { + "effect": "grant", + "id": "allow", + "label": "Allow" + }, + { + "effect": "decline", + "id": "decline", + "label": "Decline" + } + ], + "prompt": "Review the vault's requested access.", + "requester_label": "reserved:aft", + "scope": {}, + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + }, + "secret_use_answer": { + "decision": "allow", + "duration": "hour", + "envelope_sha256": "7bcd68a3ec5698ad9c9d11e0efa1d67ef7e7aa6ee18c56d963d5f6b18014f180", + "items": [ + { + "decision": "allow", + "request_id": "01HF7YAT000000000000000001" + } + ] + }, + "state": "answered", + "subject": { + "agent_id": "agent-A", + "head_agent_id": "agent-A", + "label": "agent-A" + }, + "target": { + "kind": "secret_use", + "label": "Vault access" + }, + "title": "Secret use approval", + "urgency": "normal" + } + } +} diff --git a/test-vectors/cingulate-secret-use-approval-v1/filer-await-declined.json b/test-vectors/cingulate-secret-use-approval-v1/filer-await-declined.json new file mode 100644 index 0000000000..873b63f3e1 --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/filer-await-declined.json @@ -0,0 +1,67 @@ +{ + "method": "consent.await", + "params": { + "elicitation_id": "el_1", + "timeout_ms": 0 + }, + "reply": { + "result": { + "answered_at_ms": 1700000000000, + "answered_choice_id": "decline", + "answered_device": "11111111111111111111111111111111", + "created_at_ms": 1700000000000, + "default": "decline", + "elicitation_id": "el_1", + "expires_at_ms": 1700000600000, + "facts": [], + "kind": "secret_use", + "late_execution": "notify_only", + "material_damage": false, + "on_expiry": "deny", + "options": [ + { + "effect": "grant", + "id": "allow", + "label": "Allow" + }, + { + "effect": "decline", + "id": "decline", + "label": "Decline" + } + ], + "prompt": "Review the vault's requested access.", + "requester_label": "reserved:aft", + "scope": {}, + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + }, + "secret_use_answer": { + "decision": "decline", + "items": [ + { + "decision": "decline", + "request_id": "01HF7YAT000000000000000001" + } + ] + }, + "state": "answered", + "subject": { + "agent_id": "agent-A", + "head_agent_id": "agent-A", + "label": "agent-A" + }, + "target": { + "kind": "secret_use", + "label": "Vault access" + }, + "title": "Secret use approval", + "urgency": "normal" + } + } +} diff --git a/test-vectors/cingulate-secret-use-approval-v1/filer-await-expired.json b/test-vectors/cingulate-secret-use-approval-v1/filer-await-expired.json new file mode 100644 index 0000000000..bbb7dc9eee --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/filer-await-expired.json @@ -0,0 +1,56 @@ +{ + "method": "consent.await", + "params": { + "elicitation_id": "el_1", + "timeout_ms": 0 + }, + "reply": { + "result": { + "answered_at_ms": 1700000600000, + "created_at_ms": 1700000000000, + "default": "decline", + "elicitation_id": "el_1", + "expires_at_ms": 1700000600000, + "facts": [], + "kind": "secret_use", + "late_execution": "notify_only", + "material_damage": false, + "on_expiry": "deny", + "options": [ + { + "effect": "grant", + "id": "allow", + "label": "Allow" + }, + { + "effect": "decline", + "id": "decline", + "label": "Decline" + } + ], + "prompt": "Review the vault's requested access.", + "requester_label": "reserved:aft", + "scope": {}, + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + }, + "state": "expired", + "subject": { + "agent_id": "agent-A", + "head_agent_id": "agent-A", + "label": "agent-A" + }, + "target": { + "kind": "secret_use", + "label": "Vault access" + }, + "title": "Secret use approval", + "urgency": "normal" + } + } +} diff --git a/test-vectors/cingulate-secret-use-approval-v1/filer-await-partial.json b/test-vectors/cingulate-secret-use-approval-v1/filer-await-partial.json new file mode 100644 index 0000000000..4358369bdc --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/filer-await-partial.json @@ -0,0 +1,77 @@ +{ + "method": "consent.await", + "params": { + "elicitation_id": "el_1", + "timeout_ms": 0 + }, + "reply": { + "result": { + "answered_at_ms": 1700000000000, + "answered_choice_id": "allow", + "answered_device": "11111111111111111111111111111111", + "created_at_ms": 1700000000000, + "default": "decline", + "elicitation_id": "el_1", + "expires_at_ms": 1700000600000, + "facts": [], + "kind": "secret_use", + "late_execution": "notify_only", + "material_damage": false, + "on_expiry": "deny", + "options": [ + { + "effect": "grant", + "id": "allow", + "label": "Allow" + }, + { + "effect": "decline", + "id": "decline", + "label": "Decline" + } + ], + "prompt": "Review the vault's requested access.", + "requester_label": "reserved:aft", + "scope": {}, + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + }, + { + "request_id": "01HF7YAT000000000000000002", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMiIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + }, + "secret_use_answer": { + "decision": "allow", + "duration": "hour", + "envelope_sha256": "7bcd68a3ec5698ad9c9d11e0efa1d67ef7e7aa6ee18c56d963d5f6b18014f180", + "items": [ + { + "decision": "allow", + "request_id": "01HF7YAT000000000000000001" + }, + { + "decision": "decline", + "request_id": "01HF7YAT000000000000000002" + } + ] + }, + "state": "answered", + "subject": { + "agent_id": "agent-A", + "head_agent_id": "agent-A", + "label": "agent-A" + }, + "target": { + "kind": "secret_use", + "label": "Vault access" + }, + "title": "Secret use approval", + "urgency": "normal" + } + } +} diff --git a/test-vectors/cingulate-secret-use-approval-v1/filer-await-pending.json b/test-vectors/cingulate-secret-use-approval-v1/filer-await-pending.json new file mode 100644 index 0000000000..06b0e978a5 --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/filer-await-pending.json @@ -0,0 +1,55 @@ +{ + "method": "consent.await", + "params": { + "elicitation_id": "el_1", + "timeout_ms": 0 + }, + "reply": { + "result": { + "created_at_ms": 1700000000000, + "default": "decline", + "elicitation_id": "el_1", + "expires_at_ms": 1700000600000, + "facts": [], + "kind": "secret_use", + "late_execution": "notify_only", + "material_damage": false, + "on_expiry": "deny", + "options": [ + { + "effect": "grant", + "id": "allow", + "label": "Allow" + }, + { + "effect": "decline", + "id": "decline", + "label": "Decline" + } + ], + "prompt": "Review the vault's requested access.", + "requester_label": "reserved:aft", + "scope": {}, + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + }, + "state": "pending", + "subject": { + "agent_id": "agent-A", + "head_agent_id": "agent-A", + "label": "agent-A" + }, + "target": { + "kind": "secret_use", + "label": "Vault access" + }, + "title": "Secret use approval", + "urgency": "normal" + } + } +} diff --git a/test-vectors/cingulate-secret-use-approval-v1/filer-refusals.json b/test-vectors/cingulate-secret-use-approval-v1/filer-refusals.json new file mode 100644 index 0000000000..0f67424cc0 --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/filer-refusals.json @@ -0,0 +1,92 @@ +{ + "cases": [ + { + "method": "consent.request", + "name": "agent-mismatch", + "params": { + "kind": "secret_use", + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + } + }, + "reply": { + "error": { + "code": "elicitation_invalid_request", + "message": "secret use agent_id must match the stamped agent_id" + } + } + }, + { + "method": "consent.request", + "name": "non-canonical", + "params": { + "kind": "secret_use", + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "ewogICJhZ2VudF9pZCI6ICJhZ2VudC1BIiwKICAiY292ZXJhZ2UiOiAic2NvcGUiLAogICJkdXJhdGlvbnMiOiBbCiAgICAib25jZSIsCiAgICAiaG91ciIsCiAgICAic2Vzc2lvbiIsCiAgICAicGVybWFuZW50IgogIF0sCiAgIm1lbWJlcl9pZCI6ICJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLAogICJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOiAxNzAwMDAwNjAwMDAwLAogICJyZXF1ZXN0X2lkIjogIjAxSEY3WUFUMDAwMDAwMDAwMDAwMDAwMDAxIiwKICAic2NvcGUiOiB7CiAgICAiZXBvY2giOiA3LAogICAgImtpbmQiOiAiaGVhZCIsCiAgICAib3duZXIiOiAicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwKICAgICJyZWYiOiAiSCIKICB9LAogICJzZWNyZXQiOiB7CiAgICAiaW5jYXJuYXRpb24iOiAiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAxNDkiLAogICAgImtpbmQiOiAiYWdlbnRfYXBpX2tleSIsCiAgICAibmFtZSI6ICJrZXkiCiAgfSwKICAidXNlIjogewogICAgImhvc3QiOiAieC50ZXN0IiwKICAgICJsb2NhdGlvbiI6ICJoZWFkZXI6WC1BcGktS2V5IiwKICAgICJwYXRoX3ByZWZpeCI6ICIvcmVwb3MvYSIKICB9LAogICJ2IjogImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIgp9" + } + ] + } + }, + "reply": { + "error": { + "code": "elicitation_invalid_request", + "message": "invalid secret approval statement or envelope" + } + } + }, + { + "method": "consent.request", + "name": "request-conflict", + "params": { + "kind": "secret_use", + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + }, + { + "request_id": "01HF7YAT000000000000000002", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMiIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + } + }, + "reply": { + "error": { + "code": "elicitation_request_conflict", + "message": "elicitation_request_conflict: secret request id is already used" + } + } + }, + { + "method": "consent.request", + "name": "broca-refused", + "params": { + "kind": "secret_use", + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + } + }, + "reply": { + "error": { + "code": "elicitation_requester_refused", + "message": "requester reserved:broca is refused" + } + } + } + ] +} diff --git a/test-vectors/cingulate-secret-use-approval-v1/filer-request-repeat.json b/test-vectors/cingulate-secret-use-approval-v1/filer-request-repeat.json new file mode 100644 index 0000000000..c9d9c2fbf1 --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/filer-request-repeat.json @@ -0,0 +1,19 @@ +{ + "method": "consent.request", + "params": { + "kind": "secret_use", + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + } + }, + "reply": { + "result": { + "elicitation_id": "el_1" + } + } +} diff --git a/test-vectors/cingulate-secret-use-approval-v1/filer-request.json b/test-vectors/cingulate-secret-use-approval-v1/filer-request.json new file mode 100644 index 0000000000..c9d9c2fbf1 --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/filer-request.json @@ -0,0 +1,19 @@ +{ + "method": "consent.request", + "params": { + "kind": "secret_use", + "secret_use": { + "items": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0" + } + ] + } + }, + "reply": { + "result": { + "elicitation_id": "el_1" + } + } +} diff --git a/test-vectors/cingulate-secret-use-approval-v1/secret-approval-v1.json b/test-vectors/cingulate-secret-use-approval-v1/secret-approval-v1.json new file mode 100644 index 0000000000..d09c69dfd4 --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/secret-approval-v1.json @@ -0,0 +1,111 @@ +{ + "approval_private_key_hex": "0707070707070707070707070707070707070707070707070707070707070707", + "deposit": { + "aad_b64url": "GAAAAAAAAABjay1zZWNyZXQtZGVwb3NpdC1hYWQvdjEaAAAAAAAAADAxSEY3WUFUMDAwMDAwMDAwMDAwMDAwMDAxLQAAAAAAAABmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIDAAAAAAAAAGtleQ0AAAAAAAAAYWdlbnRfYXBpX2tleQYAAAAAAAAAeC50ZXN0CAAAAAAAAAAvcmVwb3MvYRAAAAAAAAAAaGVhZGVyOlgtQXBpLUtleQ", + "ciphertext_b64url": "UDbNi7sJxIPNLrLaw9F7FEMCtsBZuuOAo1jddEYZzA3sSZRLP_kit4Rj7GBRsUKuWseHvzSgVV0R4RfyLEVrt04FUqZF0k4WQFuCbEbiroXg", + "deposit_digest": "f3630ed925db2cd603d12636bc105a6987817942c3872103f46904f9236eeea2", + "enc_b64url": "AI1P1sAGY5Vb9z3BUv5DvPE8l0U5JIckWKIp9GyAgBI", + "ephemeral_ikm_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "phone_public_hex": "5dfedd3b6bd47f6fa28ee15d969d5bb0ea53774d488bdaf9df1c6e0124b3ef22", + "phone_scalar_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "plaintext_b64url": "HgAAAAAAAABjay1zZWNyZXQtZGVwb3NpdC1wbGFpbnRleHQvdjETAAAAAAAAAGNraW9zLWZpeGVkLWFwaS1rZXk", + "recipient_public_hex": "57db4b359f23ae5e146e4e2512056704722506348c150c14753d0c933d04d421", + "recipient_scalar_hex": "0909090909090909090909090909090909090909090909090909090909090909" + }, + "vectors": [ + { + "case": "approval", + "envelope_b64url": "eyJkZWNpc2lvbiI6ImFsbG93IiwiZGV2aWNlX2ZpbmdlcnByaW50IjoiMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTEiLCJkdXJhdGlvbiI6ImhvdXIiLCJncmFudF9leHBpcmVzX2F0X21zIjoxNzAwMDAzNjAwMDAwLCJpc3N1ZWRfYXRfbXMiOjE3MDAwMDAwMDAwMDAsIml0ZW1zIjpbeyJyZXF1ZXN0X2lkIjoiMDFIRjdZQVQwMDAwMDAwMDAwMDAwMDAwMDEiLCJzdGF0ZW1lbnRfc2hhMjU2IjoiZGUzMTc0ZmViMTRjZjJlMjllYzczMDgwZGY3ODVlZTYxNTc0OWQ2MzhjYzgxYmVkOWFhZTQ4MDc5ZDBjYjgxYiJ9XSwidiI6ImNrLXNlY3JldC11c2UtZW52ZWxvcGUvdjEifQ", + "envelope_sha256": "7bcd68a3ec5698ad9c9d11e0efa1d67ef7e7aa6ee18c56d963d5f6b18014f180", + "results": [ + { + "outcome": "granted", + "request_id": "01HF7YAT000000000000000001" + } + ], + "signature_b64url": "MEUCIQDKRJk3gvG7EYBPT0xIKvcZn5bmJpnOEmY3t5fkhlkPtAIgBxhQIWr_XOuoTrazb1NrM-ZJ4FLMQhENmjQPQyuSlZw", + "statements": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0", + "statement_sha256": "de3174feb14cf2e29ec73080df785ee615749d638cc81bed9aae48079d0cb81b" + } + ], + "tampered_envelope_b64url": "eiJkZWNpc2lvbiI6ImFsbG93IiwiZGV2aWNlX2ZpbmdlcnByaW50IjoiMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTEiLCJkdXJhdGlvbiI6ImhvdXIiLCJncmFudF9leHBpcmVzX2F0X21zIjoxNzAwMDAzNjAwMDAwLCJpc3N1ZWRfYXRfbXMiOjE3MDAwMDAwMDAwMDAsIml0ZW1zIjpbeyJyZXF1ZXN0X2lkIjoiMDFIRjdZQVQwMDAwMDAwMDAwMDAwMDAwMDEiLCJzdGF0ZW1lbnRfc2hhMjU2IjoiZGUzMTc0ZmViMTRjZjJlMjllYzczMDgwZGY3ODVlZTYxNTc0OWQ2MzhjYzgxYmVkOWFhZTQ4MDc5ZDBjYjgxYiJ9XSwidiI6ImNrLXNlY3JldC11c2UtZW52ZWxvcGUvdjEifQ", + "unlisted_pending": false + }, + { + "case": "delete", + "envelope_b64url": "eyJkZWNpc2lvbiI6ImRlbGV0ZSIsImRldmljZV9maW5nZXJwcmludCI6IjExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExIiwiZHVyYXRpb24iOiJub25lIiwiZ3JhbnRfZXhwaXJlc19hdF9tcyI6MTcwMDAwMDYwMDAwMCwiaXNzdWVkX2F0X21zIjoxNzAwMDAwMDAwMDAwLCJpdGVtcyI6W3sicmVxdWVzdF9pZCI6IjAxSEY3WUFUMDAwMDAwMDAwMDAwMDAwMDAxIiwic3RhdGVtZW50X3NoYTI1NiI6ImRmNjVhZDhjZDYzYjVlYmVlODBkNTg3ZDMwNWM4ZTE5NjExMjFhMzY3MWYxZDY2NTg0MjQyNTJhZTdiZmE1ZjIifV0sInYiOiJjay1zZWNyZXQtdXNlLWVudmVsb3BlL3YxIn0", + "envelope_sha256": "61f82b0a651cb8efc9af2fe001a5a051ff09833c6abc7ad365dc6f8b8e97967a", + "results": [ + { + "outcome": "granted", + "request_id": "01HF7YAT000000000000000001" + } + ], + "signature_b64url": "MEUCIDo_hfKhCP8-j278u87PM-UKFEFb_AoGvVbbrjXugL8YAiEA6rbPE4zDISfvxqLF96TioYQoQ__NMZZ3mLqxLZdJyoU", + "statements": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNlY3JldCI6eyJpbmNhcm5hdGlvbiI6IjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMTQ5Iiwia2luZCI6ImFnZW50X2FwaV9rZXkiLCJuYW1lIjoia2V5In0sInYiOiJjay1zZWNyZXQtZGVsZXRlLXN0YXRlbWVudC92MSJ9", + "statement_sha256": "df65ad8cd63b5ebee80d587d305c8e1961121a3671f1d6658424252ae7bfa5f2" + } + ], + "tampered_envelope_b64url": "eiJkZWNpc2lvbiI6ImRlbGV0ZSIsImRldmljZV9maW5nZXJwcmludCI6IjExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExIiwiZHVyYXRpb24iOiJub25lIiwiZ3JhbnRfZXhwaXJlc19hdF9tcyI6MTcwMDAwMDYwMDAwMCwiaXNzdWVkX2F0X21zIjoxNzAwMDAwMDAwMDAwLCJpdGVtcyI6W3sicmVxdWVzdF9pZCI6IjAxSEY3WUFUMDAwMDAwMDAwMDAwMDAwMDAxIiwic3RhdGVtZW50X3NoYTI1NiI6ImRmNjVhZDhjZDYzYjVlYmVlODBkNTg3ZDMwNWM4ZTE5NjExMjFhMzY3MWYxZDY2NTg0MjQyNTJhZTdiZmE1ZjIifV0sInYiOiJjay1zZWNyZXQtdXNlLWVudmVsb3BlL3YxIn0", + "unlisted_pending": false + }, + { + "case": "batch-omitted", + "envelope_b64url": "eyJkZWNpc2lvbiI6ImFsbG93IiwiZGV2aWNlX2ZpbmdlcnByaW50IjoiMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTEiLCJkdXJhdGlvbiI6ImhvdXIiLCJncmFudF9leHBpcmVzX2F0X21zIjoxNzAwMDAzNjAwMDAwLCJpc3N1ZWRfYXRfbXMiOjE3MDAwMDAwMDAwMDAsIml0ZW1zIjpbeyJyZXF1ZXN0X2lkIjoiMDFIRjdZQVQwMDAwMDAwMDAwMDAwMDAwMDEiLCJzdGF0ZW1lbnRfc2hhMjU2IjoiZGUzMTc0ZmViMTRjZjJlMjllYzczMDgwZGY3ODVlZTYxNTc0OWQ2MzhjYzgxYmVkOWFhZTQ4MDc5ZDBjYjgxYiJ9XSwidiI6ImNrLXNlY3JldC11c2UtZW52ZWxvcGUvdjEifQ", + "envelope_sha256": "7bcd68a3ec5698ad9c9d11e0efa1d67ef7e7aa6ee18c56d963d5f6b18014f180", + "results": [ + { + "outcome": "granted", + "request_id": "01HF7YAT000000000000000001" + } + ], + "signature_b64url": "MEUCIQDKRJk3gvG7EYBPT0xIKvcZn5bmJpnOEmY3t5fkhlkPtAIgBxhQIWr_XOuoTrazb1NrM-ZJ4FLMQhENmjQPQyuSlZw", + "statements": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0", + "statement_sha256": "de3174feb14cf2e29ec73080df785ee615749d638cc81bed9aae48079d0cb81b" + }, + { + "request_id": "01HF7YAT000000000000000002", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMiIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0", + "statement_sha256": "40188343b0381787b1cce06184ac39209d964a50161cb05de515112e05fa7e91" + } + ], + "tampered_envelope_b64url": "eiJkZWNpc2lvbiI6ImFsbG93IiwiZGV2aWNlX2ZpbmdlcnByaW50IjoiMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTEiLCJkdXJhdGlvbiI6ImhvdXIiLCJncmFudF9leHBpcmVzX2F0X21zIjoxNzAwMDAzNjAwMDAwLCJpc3N1ZWRfYXRfbXMiOjE3MDAwMDAwMDAwMDAsIml0ZW1zIjpbeyJyZXF1ZXN0X2lkIjoiMDFIRjdZQVQwMDAwMDAwMDAwMDAwMDAwMDEiLCJzdGF0ZW1lbnRfc2hhMjU2IjoiZGUzMTc0ZmViMTRjZjJlMjllYzczMDgwZGY3ODVlZTYxNTc0OWQ2MzhjYzgxYmVkOWFhZTQ4MDc5ZDBjYjgxYiJ9XSwidiI6ImNrLXNlY3JldC11c2UtZW52ZWxvcGUvdjEifQ", + "unlisted_pending": true + }, + { + "case": "batch-unknown", + "envelope_b64url": "eyJkZWNpc2lvbiI6ImFsbG93IiwiZGV2aWNlX2ZpbmdlcnByaW50IjoiMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTEiLCJkdXJhdGlvbiI6ImhvdXIiLCJncmFudF9leHBpcmVzX2F0X21zIjoxNzAwMDAzNjAwMDAwLCJpc3N1ZWRfYXRfbXMiOjE3MDAwMDAwMDAwMDAsIml0ZW1zIjpbeyJyZXF1ZXN0X2lkIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLCJzdGF0ZW1lbnRfc2hhMjU2IjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCJ9LHsicmVxdWVzdF9pZCI6IjAxSEY3WUFUMDAwMDAwMDAwMDAwMDAwMDAxIiwic3RhdGVtZW50X3NoYTI1NiI6ImRlMzE3NGZlYjE0Y2YyZTI5ZWM3MzA4MGRmNzg1ZWU2MTU3NDlkNjM4Y2M4MWJlZDlhYWU0ODA3OWQwY2I4MWIifV0sInYiOiJjay1zZWNyZXQtdXNlLWVudmVsb3BlL3YxIn0", + "envelope_sha256": "8a997bb7042e0796ae75bb25a3f82ff4f06fdfc38b61ed01399f70bc61c795ee", + "results": [ + { + "code": "approval_invalid", + "outcome": "refused", + "request_id": "00000000000000000000000000" + }, + { + "outcome": "granted", + "request_id": "01HF7YAT000000000000000001" + } + ], + "signature_b64url": "MEYCIQCOfR1DCmGX7lc7j_24wxQ1APCvQv_NLjTSlqDFWXgUdgIhAPCx_LZPeLvEBsovZcsSdgF00MFhsfbi4U5iwulUJvMY", + "statements": [ + { + "request_id": "01HF7YAT000000000000000001", + "statement_b64url": "eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0", + "statement_sha256": "de3174feb14cf2e29ec73080df785ee615749d638cc81bed9aae48079d0cb81b" + } + ], + "tampered_envelope_b64url": "eiJkZWNpc2lvbiI6ImFsbG93IiwiZGV2aWNlX2ZpbmdlcnByaW50IjoiMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTEiLCJkdXJhdGlvbiI6ImhvdXIiLCJncmFudF9leHBpcmVzX2F0X21zIjoxNzAwMDAzNjAwMDAwLCJpc3N1ZWRfYXRfbXMiOjE3MDAwMDAwMDAwMDAsIml0ZW1zIjpbeyJyZXF1ZXN0X2lkIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLCJzdGF0ZW1lbnRfc2hhMjU2IjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCJ9LHsicmVxdWVzdF9pZCI6IjAxSEY3WUFUMDAwMDAwMDAwMDAwMDAwMDAxIiwic3RhdGVtZW50X3NoYTI1NiI6ImRlMzE3NGZlYjE0Y2YyZTI5ZWM3MzA4MGRmNzg1ZWU2MTU3NDlkNjM4Y2M4MWJlZDlhYWU0ODA3OWQwY2I4MWIifV0sInYiOiJjay1zZWNyZXQtdXNlLWVudmVsb3BlL3YxIn0", + "unlisted_pending": false + } + ] +} \ No newline at end of file diff --git a/test-vectors/cingulate-secret-use-approval-v1/single-phone-get.json b/test-vectors/cingulate-secret-use-approval-v1/single-phone-get.json new file mode 100644 index 0000000000..60934cefe8 --- /dev/null +++ b/test-vectors/cingulate-secret-use-approval-v1/single-phone-get.json @@ -0,0 +1 @@ +{"result":{"createdAtMs":1700000000000,"default":"decline","elicitationId":"el_1","expiresAtMs":1700000600000,"facts":[],"kind":"secret_use","lateExecution":"notify_only","materialDamage":false,"onExpiry":"deny","options":[{"effect":"grant","id":"allow","label":"Allow"},{"effect":"decline","id":"decline","label":"Decline"}],"prompt":"Review the vault's requested access.","requesterLabel":"reserved:aft","requesterName":"AFT","scope":{},"secretUse":{"items":[{"request_id":"01HF7YAT000000000000000001","statement_b64url":"eyJhZ2VudF9pZCI6ImFnZW50LUEiLCJjb3ZlcmFnZSI6InNjb3BlIiwiZHVyYXRpb25zIjpbIm9uY2UiLCJob3VyIiwic2Vzc2lvbiIsInBlcm1hbmVudCJdLCJtZW1iZXJfaWQiOiJmZWQtY29tcHV0ZXI6MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIiLCJyZXF1ZXN0X2V4cGlyZXNfYXRfbXMiOjE3MDAwMDA2MDAwMDAsInJlcXVlc3RfaWQiOiIwMUhGN1lBVDAwMDAwMDAwMDAwMDAwMDAwMSIsInNjb3BlIjp7ImVwb2NoIjo3LCJraW5kIjoiaGVhZCIsIm93bmVyIjoicmVzZXJ2ZWQ6cHJlZnJvbnRhbC1jb3JlIiwicmVmIjoiSCJ9LCJzZWNyZXQiOnsiaW5jYXJuYXRpb24iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDE0OSIsImtpbmQiOiJhZ2VudF9hcGlfa2V5IiwibmFtZSI6ImtleSJ9LCJ1c2UiOnsiaG9zdCI6IngudGVzdCIsImxvY2F0aW9uIjoiaGVhZGVyOlgtQXBpLUtleSIsInBhdGhfcHJlZml4IjoiL3JlcG9zL2EifSwidiI6ImNrLXNlY3JldC11c2Utc3RhdGVtZW50L3YxIn0"}]},"state":"pending","subject":{"agentId":"agent-A","headAgentId":"agent-A","label":"agent-A"},"target":{"kind":"secret_use","label":"Vault access"},"title":"Secret use approval","urgency":"normal"}} \ No newline at end of file From 30d64ed15699b41e1849cbc14ae8fc8aaa9a6102 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:41:45 +0100 Subject: [PATCH 038/163] mason: coordinate per-request phone approvals and proxy routing Keep requests held for the approved ten-minute window, including idle socket lifetime. Share scope/label queues, remember answered request ids until expiry, and never refile an already-known Cingulate card. Add real proxy and SDK route coverage for releases, literals, decline, expiry, timeout, concurrent/once approvals, cross-host redirects and fetch. Launch scanning is now independent of vault records; invalid released locations are evaluated only after release. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 6d75a4f40aa89a083907191db03a572b4acbd722) (cherry picked from commit 466a2f24de66f381a4e29d7859399baabf18c8ab) --- crates/aft/src/bash_background/secrets.rs | 5 +- .../bash_background/secrets_audit_tests.rs | 2 +- .../aft/src/bash_background/secrets_tests.rs | 12 +- crates/aft/src/commands/url_resolution.rs | 2 +- crates/aft/src/secret/consent.rs | 3 + crates/aft/src/secret/mod.rs | 1 + crates/aft/src/secret/requests.rs | 56 +- crates/aft/src/secret/transport.rs | 5 +- crates/aft/src/secret/transport_fake.rs | 44 +- crates/aft/src/secret/wire_tests.rs | 2 +- crates/aft/src/secret_proxy/audit_tests.rs | 2 +- crates/aft/src/secret_proxy/limits.rs | 20 + crates/aft/src/secret_proxy/mod.rs | 17 +- crates/aft/src/secret_proxy/tests.rs | 58 +- crates/aft/src/secret_proxy/vault_tests.rs | 526 ++++++++++++++++++ crates/aft/src/subc/secret_consent.rs | 94 +++- crates/aft/src/subc/secret_consent_tests.rs | 11 +- crates/aft/src/url_fetch.rs | 32 +- 18 files changed, 799 insertions(+), 93 deletions(-) create mode 100644 crates/aft/src/secret_proxy/vault_tests.rs diff --git a/crates/aft/src/bash_background/secrets.rs b/crates/aft/src/bash_background/secrets.rs index 1369855b09..7162212d6b 100644 --- a/crates/aft/src/bash_background/secrets.rs +++ b/crates/aft/src/bash_background/secrets.rs @@ -258,7 +258,9 @@ pub(crate) fn authorize( scanned_labels: labels.to_vec(), consent_context: crate::subc::current_secret_consent_context(), #[cfg(test)] - test_https_port: None, + test_https_port: TEST_HTTPS_PORT.with(std::cell::Cell::get), + #[cfg(test)] + approval_clock: None, origin: task.paths.task_id.clone(), }) .map_err(|_| refusal(id, labels, "secrets_unavailable", PROXY_UNAVAILABLE))?; @@ -423,6 +425,7 @@ fn native_roots() -> Result, ()> { #[cfg(test)] thread_local! { + static TEST_HTTPS_PORT: std::cell::Cell> = const { std::cell::Cell::new(None) }; static TEST_VAULT: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; #[cfg(unix)] static TEST_UPSTREAM: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; diff --git a/crates/aft/src/bash_background/secrets_audit_tests.rs b/crates/aft/src/bash_background/secrets_audit_tests.rs index 8dcff76792..a14ba62e78 100644 --- a/crates/aft/src/bash_background/secrets_audit_tests.rs +++ b/crates/aft/src/bash_background/secrets_audit_tests.rs @@ -495,7 +495,7 @@ fn audit_runtime_built_unscanned_label_is_refused() { std::fs::read_to_string(f.root.path().join("result")).unwrap(), "22" ); - assert_eq!(f.vault.resolves.load(Ordering::SeqCst), 1); + assert_eq!(f.vault.resolves.load(Ordering::SeqCst), 0); assert_eq!(f.vault.materials.load(Ordering::SeqCst), 0); assert!(f.seen.lock().unwrap().is_empty()); }); diff --git a/crates/aft/src/bash_background/secrets_tests.rs b/crates/aft/src/bash_background/secrets_tests.rs index ad3341e409..e147f7dceb 100644 --- a/crates/aft/src/bash_background/secrets_tests.rs +++ b/crates/aft/src/bash_background/secrets_tests.rs @@ -20,6 +20,8 @@ struct Vault { impl Vault { fn new(port: u16) -> Arc { + #[cfg(unix)] + TEST_HTTPS_PORT.with(|slot| slot.set((port != 0).then_some(port))); Arc::new(Self { resolves: AtomicUsize::new(0), materials: AtomicUsize::new(0), @@ -43,10 +45,6 @@ impl SecretVault for Vault { ) -> UseAnswer { #[cfg(unix)] { - assert!( - crate::secret_proxy::test_hardening::hardened_on_this_thread(), - "hardening must precede vault resolution" - ); // The real limits change only in the audit tests' isolated child // processes; a shared test process keeps its limits. if std::env::var_os(crate::secret_proxy::test_hardening::REAL_ENV).is_some() { @@ -901,8 +899,8 @@ assert open(sys.argv[2], 'rb').read() in open(paths[0], 'rb').read().replace(b'\ .unwrap(); assert_eq!( vault.resolves.load(Ordering::SeqCst), - index + 1, - "one resolve per distinct label, never per occurrence" + index * 2, + "launch asks nothing; each previous outgoing request asked once" ); assert_eq!( seen.load(Ordering::SeqCst), @@ -1006,7 +1004,7 @@ assert open(sys.argv[2], 'rb').read() in open(paths[0], 'rb').read().replace(b'\ assert_eq!(vault.materials.load(Ordering::SeqCst), expected_requests); assert_eq!( *vault.resolve_kinds.lock().unwrap(), - ["worker", "head"].repeat(expected_requests / 4) + ["worker", "worker", "head", "head"].repeat(expected_requests / 4) ); assert_eq!( *vault.material_kinds.lock().unwrap(), diff --git a/crates/aft/src/commands/url_resolution.rs b/crates/aft/src/commands/url_resolution.rs index d563978242..3675df69a9 100644 --- a/crates/aft/src/commands/url_resolution.rs +++ b/crates/aft/src/commands/url_resolution.rs @@ -1329,7 +1329,7 @@ mod url_tests { let denied = FakeVault::new(); denied.set_answer("test", FakeReply::NotFound); let run_denied = run(&fixture, &denied, &routes, &req, true); - assert!(run_denied.success); + assert!(run_denied.response.success); assert_eq!(denied.material_calls(), 0); assert_eq!( denied.audit_summary(), diff --git a/crates/aft/src/secret/consent.rs b/crates/aft/src/secret/consent.rs index 207a36ff73..df4c0bf8cd 100644 --- a/crates/aft/src/secret/consent.rs +++ b/crates/aft/src/secret/consent.rs @@ -13,6 +13,9 @@ use crate::secret::SecretScope; +#[cfg(test)] +pub(crate) static TEST_TRANSPORT_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + /// The consent kind a secret ask files under. pub const SECRET_CONSENT_KIND: &str = "secret_use"; diff --git a/crates/aft/src/secret/mod.rs b/crates/aft/src/secret/mod.rs index 21a30b7573..4026cc4983 100644 --- a/crates/aft/src/secret/mod.rs +++ b/crates/aft/src/secret/mod.rs @@ -14,6 +14,7 @@ pub mod consent; pub mod eligibility; pub mod outcome; +#[cfg(unix)] pub(crate) mod requests; pub mod scope; pub mod transport; diff --git a/crates/aft/src/secret/requests.rs b/crates/aft/src/secret/requests.rs index cddf1ab25e..bce993036e 100644 --- a/crates/aft/src/secret/requests.rs +++ b/crates/aft/src/secret/requests.rs @@ -3,7 +3,8 @@ use std::collections::{BTreeMap, HashMap}; use std::future::Future; use std::pin::Pin; -use std::sync::{Arc, LazyLock, Mutex, Weak}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex}; use std::time::Duration; use super::{ApiKeyTarget, SecretConsentRequest, SecretScope, SecretVault, UseAnswer}; @@ -29,9 +30,13 @@ struct History { filed: Option<(String, u64)>, used_after_card: bool, } -type Slot = tokio::sync::Mutex; -type SlotKey = (SecretScope, String); -static SLOTS: LazyLock>>> = LazyLock::new(Mutex::default); +#[derive(Default)] +struct Slot { + history: tokio::sync::Mutex, + expires: AtomicU64, +} +type SlotKey = (String, String, u64, String); +static SLOTS: LazyLock>>> = LazyLock::new(Mutex::default); pub(crate) struct RequestSecrets { vault: Arc, @@ -40,6 +45,7 @@ pub(crate) struct RequestSecrets { slots: Mutex>>, status: Mutex>>, clock: Arc, + active: Arc, } impl RequestSecrets { @@ -55,6 +61,7 @@ impl RequestSecrets { slots: Mutex::default(), status: Mutex::default(), clock: Arc::new(WallClock), + active: Arc::default(), } } @@ -69,13 +76,21 @@ impl RequestSecrets { let mut slots = SLOTS .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - slots.retain(|_, slot| slot.strong_count() > 0); - let key = (self.scope.clone(), label.to_owned()); - if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) { - return slot; + slots.retain(|_, slot| { + Arc::strong_count(slot) > 1 + || slot.expires.load(Ordering::Acquire) > self.clock.now_ms() + }); + let key = ( + self.scope.owner().into(), + self.scope.reference().into(), + self.scope.stamp().scope_epoch, + label.to_owned(), + ); + if let Some(slot) = slots.get(&key) { + return slot.clone(); } let slot = Arc::new(Slot::default()); - slots.insert(key, Arc::downgrade(&slot)); + slots.insert(key, slot.clone()); slot }) .clone() @@ -91,11 +106,31 @@ impl RequestSecrets { .collect() } + pub(crate) fn activity(&self) -> Arc { + self.active.clone() + } + + #[cfg(test)] + pub(crate) fn with_clock(mut self, clock: Option>) -> Self { + if let Some(clock) = clock { + self.clock = clock; + } + self + } + pub(crate) async fn use_for_request( &self, label: &str, target: &ApiKeyTarget, ) -> Result { + struct Active(Arc); + impl Drop for Active { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } + } + self.active.fetch_add(1, Ordering::AcqRel); + let _active = Active(self.active.clone()); let result = tokio::select! { result = self.use_serial(label, target) => result, () = self.clock.deadline() => Err(()), @@ -123,7 +158,7 @@ impl RequestSecrets { async fn use_serial(&self, label: &str, target: &ApiKeyTarget) -> Result { let slot = self.slot(label); - let mut history = slot.lock().await; + let mut history = slot.history.lock().await; if history .filed .as_ref() @@ -164,6 +199,7 @@ impl RequestSecrets { .unwrap_or_else(std::sync::PoisonError::into_inner) .insert(label.into(), lines); history.filed = Some((pending.request_id.clone(), pending.expires_at_ms)); + slot.expires.store(pending.expires_at_ms, Ordering::Release); let (send, receive) = tokio::sync::oneshot::channel(); std::thread::spawn(move || { let _ = send.send(super::consent::hold_consent(&request)); diff --git a/crates/aft/src/secret/transport.rs b/crates/aft/src/secret/transport.rs index 81a8af00b9..6016c5ffe6 100644 --- a/crates/aft/src/secret/transport.rs +++ b/crates/aft/src/secret/transport.rs @@ -23,6 +23,9 @@ pub(crate) fn install_connection_file(path: &std::path::Path) { pub(crate) fn for_current_command() -> Option { let path = CONNECTION_FILE.read().ok()?.clone()?; + if !path.is_file() { + return None; + } let (_, session) = crate::subc::current_secret_consent_context()?; SubcSecretUseTransport::new( path, @@ -31,7 +34,7 @@ pub(crate) fn for_current_command() -> Option { ) } -/// A transport for a future request-addressed vault client. Construct it at call +/// A request-addressed vault transport. Construct it at call /// time: a standalone process has no module identity and must not open as direct. /// One command owns one consumer connection; failed connections are discarded so /// a later request can reconnect after the daemon returns. diff --git a/crates/aft/src/secret/transport_fake.rs b/crates/aft/src/secret/transport_fake.rs index 3d40f55b6e..a351d5b0e7 100644 --- a/crates/aft/src/secret/transport_fake.rs +++ b/crates/aft/src/secret/transport_fake.rs @@ -37,6 +37,7 @@ fn invalid() -> Value { pub(crate) struct State { pub opens: Vec, pub calls: usize, + pub params: Vec, pub connections: usize, pub hold_open: bool, pub hold_reply: bool, @@ -45,10 +46,21 @@ pub(crate) struct State { pub setup_incomplete: bool, pub other_pending_in_scope: usize, pub other_pending_total: usize, + pub record: Option, + pub once: bool, + pub next_request: usize, approved: HashSet, pub(super) pending: HashMap, } +pub(crate) struct Record { + pub name: String, + pub host: String, + pub path: String, + pub location: String, + pub key: Vec, +} + impl State { /// Stand in for the vault verifying the phone's signature over these bytes. pub fn approve(&mut self, statement_b64url: &str) { @@ -141,11 +153,14 @@ impl State { let matches = match ¶ms.target { Target::ApiKey(target) => { principal == "aft" - && params.name == "key" + && params.name == self.record.as_ref().map_or("key", |r| r.name.as_str()) && target.scheme == "https" && target.port.unwrap_or(443) == 443 - && target.host == "x.test" - && (target.path == "/repos/a" || target.path.starts_with("/repos/a/")) + && target.host == self.record.as_ref().map_or("x.test", |r| r.host.as_str()) + && { + let prefix = self.record.as_ref().map_or("/repos/a", |r| r.path.as_str()); + target.path == prefix || target.path.starts_with(&format!("{prefix}/")) + } } Target::Login(_) => false, }; @@ -173,7 +188,17 @@ impl State { }, ); if covered { - return json!({"result":golden(3)}); + if self.once { + self.approved.remove(&key); + self.pending.remove(&key); + } + let mut released = golden(3); + if let Some(record) = &self.record { + released["location"] = json!(record.location); + released["key_b64url"] = + json!(base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&record.key)); + } + return json!({"result":released}); } if self.setup_incomplete { return refusal("secret_setup_incomplete", "permanent"); @@ -194,7 +219,8 @@ impl State { return refusal("pending_limit", "transient"); } let mut pending = golden(5); - pending["request_id"] = json!(format!("01HF7YAT{:018}", self.pending.len() + 1)); + self.next_request += 1; + pending["request_id"] = json!(format!("01HF7YAT{:018}", self.next_request)); let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; let mut statement: Value = serde_json::from_slice( &engine @@ -209,6 +235,11 @@ impl State { statement["durations"] = json!(["permanent"]); } statement["agent_id"] = json!(stamp.attributes.agent_id); + if let Some(record) = &self.record { + statement["secret"]["name"] = json!(record.name); + statement["use"] = + json!({"host":record.host, "path_prefix":record.path, "location":record.location}); + } pending["statement_b64url"] = json!(engine.encode(serde_json::to_vec(&statement).unwrap())); self.pending.insert(key, pending.clone()); json!({"result":pending}) @@ -362,7 +393,8 @@ impl FakeClaustrum { } else { assert_eq!(value["method"], "secret.use"); let mut state = state.lock().unwrap(); - state.calls += 1; + state.calls += 1; + state.params.push(value["params"].clone()); if state.disconnect { break; } if state.hold_reply { continue; } let reply = state.serve(&principal, value["params"].clone()); diff --git a/crates/aft/src/secret/wire_tests.rs b/crates/aft/src/secret/wire_tests.rs index ec4f7545bb..5a694f7423 100644 --- a/crates/aft/src/secret/wire_tests.rs +++ b/crates/aft/src/secret/wire_tests.rs @@ -442,7 +442,7 @@ fn the_secret_use_client_sends_the_pinned_params_and_reads_a_release() { let UseAnswer::Released(released) = answer else { panic!("a release keeps material for this request only"); }; - assert_eq!(released.location(), "header:X-Api-Key"); + assert_eq!(released.location, "header:X-Api-Key"); assert_eq!(released.material(), b"sk-example"); let sent = transport.0.lock().unwrap(); assert_eq!(sent.len(), 1); diff --git a/crates/aft/src/secret_proxy/audit_tests.rs b/crates/aft/src/secret_proxy/audit_tests.rs index 90b74440ac..3d95dea70d 100644 --- a/crates/aft/src/secret_proxy/audit_tests.rs +++ b/crates/aft/src/secret_proxy/audit_tests.rs @@ -331,7 +331,7 @@ async fn audit_unscanned_header_refusal() { async fn audit_ipv6_literal() { let server = Server::new("ipv6-h1").await; let vault = Vault::new(); - let mut cfg = config( + let cfg = config( vault.clone(), server.address.port(), &[("test", Location::Bearer)], diff --git a/crates/aft/src/secret_proxy/limits.rs b/crates/aft/src/secret_proxy/limits.rs index 2ea148497c..abc1e6f230 100644 --- a/crates/aft/src/secret_proxy/limits.rs +++ b/crates/aft/src/secret_proxy/limits.rs @@ -18,6 +18,7 @@ pub(super) struct IdleIo { inner: T, timeout: Duration, timer: Pin>, + approval_waits: Option>, } impl IdleIo { @@ -26,9 +27,28 @@ impl IdleIo { inner, timeout, timer: Box::pin(tokio::time::sleep(timeout)), + approval_waits: None, } } + pub(super) fn with_approval_waits( + mut self, + waits: std::sync::Arc, + ) -> Self { + self.approval_waits = Some(waits); + self + } fn expired(&mut self, cx: &mut Context<'_>) -> bool { + // A held approval has its own ten-minute deadline. Quiet sockets must + // remain usable long enough to return the card's result to the client. + if self + .approval_waits + .as_ref() + .is_some_and(|waits| waits.load(std::sync::atomic::Ordering::Acquire) != 0) + { + let _ = self.timer.as_mut().poll(cx); + self.touch(); + return false; + } self.timer.as_mut().poll(cx).is_ready() } fn touch(&mut self) { diff --git a/crates/aft/src/secret_proxy/mod.rs b/crates/aft/src/secret_proxy/mod.rs index 0ba47f17f6..4107ecee3f 100644 --- a/crates/aft/src/secret_proxy/mod.rs +++ b/crates/aft/src/secret_proxy/mod.rs @@ -97,6 +97,8 @@ pub struct ProxyConfig { pub consent_context: Option<(std::path::PathBuf, String)>, #[cfg(test)] pub test_https_port: Option, + #[cfg(test)] + pub approval_clock: Option>, pub origin: String, } @@ -379,12 +381,15 @@ impl SecretProxy { let token = Zeroizing::new(base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&random[..])); let expected_auth = basic_credentials(&token)?; + let approvals = crate::secret::requests::RequestSecrets::new( + config.vault.clone(), + config.authorization_scope.clone(), + config.consent_context.clone(), + ); + #[cfg(test)] + let approvals = approvals.with_clock(config.approval_clock.clone()); let state = Arc::new(State { - approvals: crate::secret::requests::RequestSecrets::new( - config.vault.clone(), - config.authorization_scope.clone(), - config.consent_context.clone(), - ), + approvals, config, session: AtomicI32::new(0), session_birth: std::sync::atomic::AtomicU64::new(0), @@ -535,7 +540,7 @@ async fn serve( connections.spawn(async move { let session = state.session.load(Ordering::Acquire); if session <= 0 || !state.generation_valid() || !peer::authorized(&socket, session) || !state.generation_valid() { return; } - let socket = limits::IdleIo::new(socket, state.idle_timeout()); + let socket = limits::IdleIo::new(socket, state.idle_timeout()).with_approval_waits(state.approvals.activity()); let service = service_fn(move |request| connect(request, state.clone(), lease.clone())); let _ = hyper::server::conn::http1::Builder::new() .serve_connection(TokioIo::new(socket), service).with_upgrades().await; diff --git a/crates/aft/src/secret_proxy/tests.rs b/crates/aft/src/secret_proxy/tests.rs index 838b1b256a..1d46fe5c66 100644 --- a/crates/aft/src/secret_proxy/tests.rs +++ b/crates/aft/src/secret_proxy/tests.rs @@ -133,6 +133,7 @@ fn config(vault: Arc, port: u16, locations: &[(&str, Location)]) -> Proxy scanned_labels: locations.iter().map(|(l, _)| (*l).to_owned()).collect(), consent_context: None, test_https_port: Some(port), + approval_clock: None, origin: "test-task".into(), } } @@ -274,6 +275,25 @@ impl Server { .lock() .push(json!({"target":parts.uri.to_string(),"headers":fields, "body":String::from_utf8_lossy(&data)})); + if mode == "redirect" + && parts + .headers + .get("host") + .is_some_and(|v| v.as_bytes().starts_with(b"localhost:")) + { + let host = parts.headers["host"].to_str().unwrap(); + let port = host.rsplit_once(':').unwrap().1; + let mut response = Response::new(full(Bytes::new())); + *response.status_mut() = StatusCode::FOUND; + response.headers_mut().insert( + "location", + HeaderValue::from_str(&format!( + "https://127.0.0.1:{port}/approved" + )) + .unwrap(), + ); + return Ok::<_, std::convert::Infallible>(response); + } if let Some(upgrade) = upgrade { let mut response = Response::new(full(Bytes::new())); *response.status_mut() = StatusCode::SWITCHING_PROTOCOLS; @@ -689,6 +709,9 @@ fn client_child() { #[path = "audit_tests.rs"] mod audit_tests; +#[path = "vault_tests.rs"] +mod vault_tests; + fn child_reply(line: &str) -> Option { // With a single test thread, libtest prefixes uncaptured output with the // test name on the same line. Only the following JSON is our IPC reply. @@ -1269,7 +1292,7 @@ async fn proxy_query_and_json_escape_material_without_partial_header_writes() { async fn proxy_unavailable_release_refuses_and_tunnel_closes_on_stop() { let server = Server::new("echo-request").await; let vault = Vault::new(); - let mut cfg = config( + let cfg = config( vault.clone(), server.address.port(), &[("test", Location::Bearer)], @@ -1408,31 +1431,24 @@ async fn proxy_requires_armed_session_token_and_canonical_connect_authority() { } #[test] -fn proxy_only_accepts_grants_for_the_exact_launch_scan() { +fn proxy_only_accepts_a_nonempty_unique_valid_launch_scan() { let vault = Vault::new(); let make = || config(vault.clone(), 443, &[("test", Location::Bearer)]); let tls = rustls::ClientConfig::builder() .with_root_certificates(rustls::RootCertStore::empty()) .with_no_client_auth(); - let mut cfg = make(); - cfg.scanned_labels.clear(); - assert_eq!( - SecretProxy::start_with_tls(cfg, tls.clone()).err(), - Some(ProxyError::InvalidRules) - ); - let mut cfg = make(); - cfg.scanned_labels.push("file-label".into()); - assert_eq!( - SecretProxy::start_with_tls(cfg, tls.clone()).err(), - Some(ProxyError::InvalidRules) - ); - let mut cfg = make(); - vault.rules.lock().get_mut("test").unwrap().location = - Location::Header("Accept-Encoding".into()); - assert_eq!( - SecretProxy::start_with_tls(cfg, tls).err(), - Some(ProxyError::InvalidRules) - ); + for labels in [ + vec![], + vec!["test".into(), "test".into()], + vec!["bad label".into()], + ] { + let mut cfg = make(); + cfg.scanned_labels = labels; + assert_eq!( + SecretProxy::start_with_tls(cfg, tls.clone()).err(), + Some(ProxyError::InvalidRules) + ); + } } /// A stopped proxy's listener is closed before `stop()` returns, so a fresh diff --git a/crates/aft/src/secret_proxy/vault_tests.rs b/crates/aft/src/secret_proxy/vault_tests.rs new file mode 100644 index 0000000000..0a26213468 --- /dev/null +++ b/crates/aft/src/secret_proxy/vault_tests.rs @@ -0,0 +1,526 @@ +//! Real proxy requests backed by the attested consumer route and a phone double. + +use super::*; +use crate::secret::transport::fake::{FakeClaustrum, Record, NONCE}; +use crate::secret::{ + ConsentOutcome, SecretConsentRequest, SecretConsentTransport, SubcSecretVault, +}; +use std::sync::atomic::AtomicU64; + +struct Clock { + now: Arc, + ticks: tokio::sync::watch::Sender, +} + +impl Clock { + fn new() -> Arc { + let (ticks, _) = tokio::sync::watch::channel(1_700_000_000_000); + Arc::new(Self { + now: Arc::new(AtomicU64::new(1_700_000_000_000)), + ticks, + }) + } + fn advance(&self, ms: u64) { + let now = self.now.fetch_add(ms, Ordering::SeqCst) + ms; + self.ticks.send_replace(now); + } +} + +impl crate::secret::requests::ApprovalClock for Clock { + fn now_ms(&self) -> u64 { + self.now.load(Ordering::SeqCst) + } + fn deadline(&self) -> std::pin::Pin + Send>> { + let end = self.now.load(Ordering::SeqCst) + 600_000; + let mut ticks = self.ticks.subscribe(); + Box::pin(async move { + while *ticks.borrow_and_update() < end { + if ticks.changed().await.is_err() { + break; + } + } + }) + } +} + +#[derive(Default)] +struct PhoneState { + requests: Vec, + answers: HashMap, + closed: bool, +} + +struct Phone { + state: std::sync::Mutex, + answer: std::sync::Condvar, + filed: tokio::sync::Notify, +} + +impl Phone { + fn new() -> Arc { + Arc::new(Self { + state: Default::default(), + answer: Default::default(), + filed: Default::default(), + }) + } + async fn card(&self, count: usize) -> SecretConsentRequest { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let notified = self.filed.notified(); + if let Some(request) = self.state.lock().unwrap().requests.get(count - 1).cloned() { + return request; + } + notified.await; + } + }) + .await + .expect("the proxy filed the expected card") + } + fn decide(&self, daemon: &FakeClaustrum, count: usize, answer: ConsentOutcome) { + let mut state = self.state.lock().unwrap(); + let request = &state.requests[count - 1]; + if answer == ConsentOutcome::Answered { + daemon + .state + .lock() + .unwrap() + .approve(&request.statement_b64url); + } + let id = request.request_id.clone(); + state.answers.insert(id, answer); + self.answer.notify_all(); + } + fn close(&self) { + self.state.lock().unwrap().closed = true; + self.answer.notify_all(); + } +} + +impl SecretConsentTransport for Phone { + fn file(&self, request: &SecretConsentRequest) -> ConsentOutcome { + self.hold(request) + } + fn hold(&self, request: &SecretConsentRequest) -> ConsentOutcome { + let mut state = self.state.lock().unwrap(); + assert!( + !state + .requests + .iter() + .any(|r| r.request_id == request.request_id), + "never file the same vault id twice" + ); + state.requests.push(request.clone()); + self.filed.notify_one(); + loop { + if state.closed { + return ConsentOutcome::Expired; + } + if let Some(answer) = state.answers.get(&request.request_id) { + return answer.clone(); + } + state = self.answer.wait(state).unwrap(); + } + } + fn withhold(&self, _: &SecretConsentRequest) -> ConsentOutcome { + ConsentOutcome::Expired + } +} + +struct PhoneGuard { + phone: Arc, + _lock: std::sync::MutexGuard<'static, ()>, +} + +impl PhoneGuard { + fn install() -> Self { + let lock = crate::secret::consent::TEST_TRANSPORT_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let phone = Phone::new(); + crate::secret::consent::install_transport(Some(phone.clone())); + Self { phone, _lock: lock } + } +} +impl Drop for PhoneGuard { + fn drop(&mut self) { + self.phone.close(); + crate::secret::consent::install_transport(None); + } +} + +fn record(location: &str) -> Record { + Record { + name: "test".into(), + host: "localhost".into(), + path: "/approved".into(), + location: location.into(), + key: K1.to_vec(), + } +} + +fn live_config(daemon: &FakeClaustrum, port: u16, clock: Arc) -> ProxyConfig { + static SEQUENCE: AtomicU64 = AtomicU64::new(1); + let auth = scope(&format!( + "live-vault-{}", + SEQUENCE.fetch_add(1, Ordering::SeqCst) + )); + let transport = crate::secret::transport::SubcSecretUseTransport::new( + daemon.path.clone(), + subc_protocol::BindIdentity::new("test", "session"), + Duration::from_secs(2), + ) + .unwrap(); + ProxyConfig { + vault: Arc::new(SubcSecretVault::new(Arc::new(transport))), + authorization_scope: auth, + lifetime_scope: scope("run"), + scanned_labels: vec!["test".into()], + consent_context: Some((std::path::PathBuf::from("project"), "session".into())), + test_https_port: Some(port), + approval_clock: Some(clock), + origin: "live-proxy-task".into(), + } +} + +fn request_in_thread( + mut client: Client, + overlap: bool, +) -> std::thread::JoinHandle<(Value, Client)> { + std::thread::spawn(move || { + let reply = if overlap { + client.command(json!({"op":"overlap"})) + } else { + client.request( + "/approved", + json!({"authorization":format!("Bearer {MARKER}")}), + "", + ) + }; + (reply, client) + }) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_vault_release_injects_only_the_deposit_location_and_every_use_is_audited() { + let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); + let daemon = FakeClaustrum::start(); + daemon.state.lock().unwrap().record = Some(record("authorization_bearer")); + let phone = PhoneGuard::install(); + let server = Server::new("echo-request").await; + let cfg = live_config(&daemon, server.address.port(), Clock::new()); + let expected_scope = cfg.authorization_scope.clone(); + let proxy = SecretProxy::start_with_tls(cfg, server.client_tls.clone()).unwrap(); + let mut client = Client::spawn(&proxy, &server); + proxy.arm_command(client.process.id()).unwrap(); + client.connect(false); + let worker = request_in_thread(client, false); + let card = phone.phone.card(1).await; + assert_eq!(card.scope, expected_scope); + assert!( + server.seen.lock().is_empty(), + "approval holds before any upstream request" + ); + assert_eq!(proxy.approval_status(), vec!["secret test: waiting for approval on the phone for localhost (requests wait up to 10 minutes)"]); + phone.phone.decide(&daemon, 1, ConsentOutcome::Answered); + let (reply, mut client) = worker.join().unwrap(); + assert_eq!(reply["status"], 200); + assert!(!reply.to_string().contains(std::str::from_utf8(K1).unwrap())); + let next = client.request( + &format!("/approved/next?key={MARKER}"), + json!({"authorization":format!("Bearer {MARKER}"), "x-literal":MARKER}), + "", + ); + assert_eq!(next["status"], 200); + let seen = server.seen.lock(); + assert_eq!(seen.len(), 2); + assert_eq!( + seen[1]["headers"]["authorization"], + format!("Bearer {}", std::str::from_utf8(K1).unwrap()) + ); + assert_eq!(seen[1]["headers"]["x-literal"], MARKER); + assert!(seen[1]["target"].as_str().unwrap().contains(MARKER)); + let state = daemon.state.lock().unwrap(); + assert_eq!(state.calls, 3, "initial ask, approved retry, next request"); + assert_eq!( + state.connections, 1, + "one command owns one reused daemon consumer" + ); + assert_eq!( + state.params[2]["target"], + json!({"scheme":"https", "host":"localhost", "port":443, "path":"/approved/next"}) + ); + assert_eq!(phone.phone.state.lock().unwrap().requests.len(), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_vault_decline_and_expiry_answer_403_without_sending_or_retrying() { + let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); + for decision in [ConsentOutcome::Declined, ConsentOutcome::Expired] { + let daemon = FakeClaustrum::start(); + daemon.state.lock().unwrap().record = Some(record("authorization_bearer")); + let phone = PhoneGuard::install(); + let server = Server::new("echo-request").await; + let proxy = SecretProxy::start_with_tls( + live_config(&daemon, server.address.port(), Clock::new()), + server.client_tls.clone(), + ) + .unwrap(); + let mut client = Client::spawn(&proxy, &server); + proxy.arm_command(client.process.id()).unwrap(); + client.connect(false); + let worker = request_in_thread(client, false); + phone.phone.card(1).await; + phone.phone.decide(&daemon, 1, decision); + let (reply, mut client) = worker.join().unwrap(); + assert_eq!(reply["status"], 403); + assert_eq!( + reply["body"], + "secret test was not approved for localhost; the request was not sent" + ); + assert!(server.seen.lock().is_empty()); + assert_eq!( + daemon.state.lock().unwrap().calls, + 1, + "an answered decline/expiry never retries" + ); + let next = client.request( + "/approved", + json!({"authorization":format!("Bearer {MARKER}")}), + "", + ); + assert_eq!( + next["status"], 403, + "a matching retry sees the same unanswered vault id" + ); + assert_eq!( + phone.phone.state.lock().unwrap().requests.len(), + 1, + "the same request_id is never refiled" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_vault_injected_ten_minute_deadline_answers_403_locally() { + let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); + let daemon = FakeClaustrum::start(); + daemon.state.lock().unwrap().record = Some(record("authorization_bearer")); + let phone = PhoneGuard::install(); + let clock = Clock::new(); + let server = Server::new("echo-request").await; + let proxy = SecretProxy::start_with_tls( + live_config(&daemon, server.address.port(), clock.clone()), + server.client_tls.clone(), + ) + .unwrap(); + let mut client = Client::spawn(&proxy, &server); + proxy.arm_command(client.process.id()).unwrap(); + client.connect(false); + let worker = request_in_thread(client, false); + phone.phone.card(1).await; + clock.advance(599_999); + assert!( + !worker.is_finished(), + "the full ten-minute window remains available" + ); + clock.advance(1); + let (reply, _client) = worker.join().unwrap(); + assert_eq!(reply["status"], 403); + assert_eq!( + reply["body"], + "secret test was not approved for localhost; the request was not sent" + ); + assert!(server.seen.lock().is_empty()); + assert!(proxy.approval_status().is_empty()); + phone.phone.close(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_vault_concurrent_requests_share_one_card_and_once_queues_cards_serially() { + let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); + for once in [false, true] { + let daemon = FakeClaustrum::start(); + { + let mut state = daemon.state.lock().unwrap(); + state.record = Some(record("authorization_bearer")); + state.once = once; + } + let phone = PhoneGuard::install(); + let server = Server::new("echo-request").await; + let proxy = SecretProxy::start_with_tls( + live_config(&daemon, server.address.port(), Clock::new()), + server.client_tls.clone(), + ) + .unwrap(); + let mut client = Client::spawn(&proxy, &server); + proxy.arm_command(client.process.id()).unwrap(); + client.connect(true); + let worker = request_in_thread(client, true); + let first = phone.phone.card(1).await; + assert!(server.seen.lock().is_empty()); + assert_eq!(phone.phone.state.lock().unwrap().requests.len(), 1); + phone.phone.decide(&daemon, 1, ConsentOutcome::Answered); + if once { + let second = phone.phone.card(2).await; + assert_ne!(first.request_id, second.request_id); + assert!(!worker.is_finished()); + assert!(proxy.approval_status().iter().any(|line| line + == "secret test: the approval covered one request; another card was sent")); + phone.phone.decide(&daemon, 2, ConsentOutcome::Answered); + } + let (reply, _client) = worker.join().unwrap(); + assert!( + reply.get("a").is_some(), + "both HTTP/2 requests completed: {reply}" + ); + assert!(!reply.to_string().contains(std::str::from_utf8(K1).unwrap())); + assert_eq!(server.seen.lock().len(), 2); + assert_eq!( + phone.phone.state.lock().unwrap().requests.len(), + if once { 2 } else { 1 } + ); + assert_eq!(daemon.state.lock().unwrap().calls, if once { 4 } else { 3 }); + assert_eq!(daemon.state.lock().unwrap().connections, 1); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_vault_wrong_host_is_literal_and_never_files_a_card() { + let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); + let daemon = FakeClaustrum::start(); + let mut other = record("authorization_bearer"); + other.host = "other.test".into(); + daemon.state.lock().unwrap().record = Some(other); + let phone = PhoneGuard::install(); + let server = Server::new("echo-request").await; + let proxy = SecretProxy::start_with_tls( + live_config(&daemon, server.address.port(), Clock::new()), + server.client_tls.clone(), + ) + .unwrap(); + let mut client = Client::spawn(&proxy, &server); + proxy.arm_command(client.process.id()).unwrap(); + client.connect(false); + assert_eq!( + client.request( + "/approved", + json!({"authorization":format!("Bearer {MARKER}")}), + "" + )["status"], + 200 + ); + assert_eq!( + server.seen.lock()[0]["headers"]["authorization"], + format!("Bearer {MARKER}") + ); + assert_eq!(daemon.state.lock().unwrap().calls, 1); + assert!(phone.phone.state.lock().unwrap().requests.is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_vault_redirect_to_another_host_asks_again_and_sends_no_key() { + let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); + let daemon = FakeClaustrum::start(); + daemon.state.lock().unwrap().record = Some(record("authorization_bearer")); + let phone = PhoneGuard::install(); + let server = Server::new("redirect").await; + let proxy = SecretProxy::start_with_tls( + live_config(&daemon, server.address.port(), Clock::new()), + server.client_tls.clone(), + ) + .unwrap(); + let mut client = Client::spawn(&proxy, &server); + proxy.arm_command(client.process.id()).unwrap(); + client.connect(false); + let worker = request_in_thread(client, false); + phone.phone.card(1).await; + phone.phone.decide(&daemon, 1, ConsentOutcome::Answered); + let (first, mut client) = worker.join().unwrap(); + assert_eq!(first["status"], 302); + let location = url::Url::parse(first["headers"]["location"].as_str().unwrap()).unwrap(); + assert_eq!(location.host_str(), Some("127.0.0.1")); + assert_eq!( + client.command( + json!({"op":"connect", "authority":format!("127.0.0.1:{}", server.address.port())}) + )["ok"], + true + ); + let second = client.request( + location.path(), + json!({"authorization":format!("Bearer {MARKER}")}), + "", + ); + assert_eq!(second["status"], 200); + assert_eq!( + server.seen.lock()[1]["headers"]["authorization"], + format!("Bearer {MARKER}") + ); + assert_eq!(daemon.state.lock().unwrap().calls, 3); + assert_eq!(phone.phone.state.lock().unwrap().requests.len(), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_vault_fetch_uses_the_concrete_target_and_holds_for_the_phone() { + let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); + let daemon = FakeClaustrum::start(); + daemon.state.lock().unwrap().record = Some(record("query:key")); + let phone = PhoneGuard::install(); + let server = Server::new("echo-request-h1").await; + let cfg = live_config(&daemon, server.address.port(), Clock::new()); + let root = tempfile::tempdir().unwrap(); + let project = root.path().to_owned(); + let options = crate::url_fetch::UrlFetchOptions { + allow_private: true, + tls_config: Some(server.client_tls.clone()), + ..Default::default() + }; + let url = format!( + "https://localhost:{}/approved/doc.md?key={MARKER}#local-fragment", + server.address.port() + ); + let port = server.address.port(); + let worker = std::thread::spawn(move || { + crate::subc::with_secret_identity_for_test( + &project, + cfg.authorization_scope.stamp().clone(), + || { + let secrets = crate::url_fetch::SecretFetch { + vault: cfg.vault.as_ref(), + authorization: Some(cfg.authorization_scope.clone()), + labels: vec!["test".into()], + test_https_port: Some(port), + call_id: "fetch-live-vault".into(), + }; + let result = + crate::url_fetch::fetch_url_with_secrets(&url, &project, options, &secrets); + result.map(|path| std::fs::read_to_string(path).unwrap()) + }, + ) + }); + phone.phone.card(1).await; + assert!(server.seen.lock().is_empty()); + phone.phone.decide(&daemon, 1, ConsentOutcome::Answered); + let document = worker + .join() + .unwrap() + .expect("the approved document was fetched"); + assert!(!document.contains(std::str::from_utf8(K1).unwrap())); + assert!(document.contains(MARKER)); + let seen = server.seen.lock(); + assert_eq!(seen.len(), 1); + assert!(seen[0]["target"] + .as_str() + .unwrap() + .contains(std::str::from_utf8(K1).unwrap())); + assert!(!seen[0]["target"] + .as_str() + .unwrap() + .contains("local-fragment")); + assert_eq!(daemon.state.lock().unwrap().calls, 2); + assert_eq!(daemon.state.lock().unwrap().connections, 1); + assert_eq!( + daemon.state.lock().unwrap().params[0]["target"], + json!({"scheme":"https", "host":"localhost", "port":443, "path":"/approved/doc.md"}) + ); +} diff --git a/crates/aft/src/subc/secret_consent.rs b/crates/aft/src/subc/secret_consent.rs index 4016576787..1709af68bc 100644 --- a/crates/aft/src/subc/secret_consent.rs +++ b/crates/aft/src/subc/secret_consent.rs @@ -5,7 +5,7 @@ //! route policy instead of in the route-agnostic `crate::secret` types. //! //! A call is eligible in exactly three forms, and every other call is -//! ineligible and never reaches `resolve`: +//! ineligible and never asks the vault: //! //! 1. **Direct Head**: a Classic call whose route carries the head agent's own //! stamp. Both scopes are that stamp. @@ -42,17 +42,28 @@ use subc_protocol::scope::ScopeStamp; const SECRET_CONSENT_TIMEOUT: Duration = Duration::from_secs(10); /// Use the same scoped daemon connection and reserved AFT filer as permission -/// asks. The production vault remains the fail-closed stand-in. +/// asks, retaining the card id so a held call never files it twice. pub(super) fn install_subc_transport(transport: Arc) { crate::secret::consent::install_transport(Some(Arc::new(SubcSecretConsent { transport, runtime: tokio::runtime::Handle::current(), + cards: Default::default(), }))); } struct SubcSecretConsent { transport: Arc, runtime: tokio::runtime::Handle, + cards: tokio::sync::Mutex< + std::collections::HashMap<(SecretScope, String), Arc>>, + >, +} + +struct Card { + created: std::time::Instant, + statement: String, + elicitation: Option, + answered: Option, } fn consent_unavailable(detail: &str) -> ConsentOutcome { @@ -128,36 +139,77 @@ impl SubcSecretConsent { } async fn file(&self, request: &SecretConsentRequest, hold: Duration) -> ConsentOutcome { - // Cingulate deduplicates the exact ordered items for this filer. Refile - // unchanged to recover the same card across retries or a lost reply, - // without refreshing its deadline or introducing a local idempotency key. - let filed = match self - .call( - request, - "consent.request", - crate::secret::request_params(request), - SECRET_CONSENT_TIMEOUT, - ) - .await - { - Ok(filed) => filed, - Err(_) => return consent_unavailable("Cingulate could not file the secret-use card"), + let card = { + let mut cards = self.cards.lock().await; + cards.retain(|_, card| { + Arc::strong_count(card) > 1 + || card + .try_lock() + .is_ok_and(|card| card.created.elapsed() < Duration::from_secs(600)) + }); + cards + .entry((request.scope.clone(), request.request_id.clone())) + .or_insert_with(|| { + Arc::new(tokio::sync::Mutex::new(Card { + created: std::time::Instant::now(), + statement: request.statement_b64url.clone(), + elicitation: None, + answered: None, + })) + }) + .clone() }; - let Some(id) = decode_secret_elicitation(&filed) else { - return consent_unavailable("Cingulate did not return a secret-use card id"); - }; - match self + let mut card = card.lock().await; + if card.statement != request.statement_b64url { + return consent_unavailable("Cingulate could not file the secret-use card"); + } + if let Some(answer) = &card.answered { + return answer.clone(); + } + if card.elicitation.is_none() { + // A lost filing reply must not cause the same request to be filed again. + card.answered = Some(consent_unavailable( + "Cingulate could not file the secret-use card", + )); + let filed = match self + .call( + request, + "consent.request", + crate::secret::request_params(request), + SECRET_CONSENT_TIMEOUT, + ) + .await + { + Ok(filed) => filed, + Err(_) => { + return consent_unavailable("Cingulate could not file the secret-use card") + } + }; + let Some(id) = decode_secret_elicitation(&filed) else { + return consent_unavailable("Cingulate did not return a secret-use card id"); + }; + card.elicitation = Some(id.to_owned()); + card.answered = None; + } + let outcome = match self .call( request, "consent.await", - serde_json::json!({"elicitation_id": id, "timeout_ms": hold.as_millis() as u64}), + serde_json::json!({"elicitation_id": card.elicitation, "timeout_ms": hold.as_millis() as u64}), hold + SECRET_CONSENT_TIMEOUT, ) .await { Ok(record) => decode_secret_record(&record, &request.request_id), Err(_) => consent_unavailable("Cingulate could not settle the secret-use card"), + }; + if matches!( + outcome, + ConsentOutcome::Answered | ConsentOutcome::Declined | ConsentOutcome::Expired + ) { + card.answered = Some(outcome.clone()); } + outcome } fn dispatch(&self, request: &SecretConsentRequest, hold: Duration) -> ConsentOutcome { diff --git a/crates/aft/src/subc/secret_consent_tests.rs b/crates/aft/src/subc/secret_consent_tests.rs index 9d6bd8327c..3720a3dba4 100644 --- a/crates/aft/src/subc/secret_consent_tests.rs +++ b/crates/aft/src/subc/secret_consent_tests.rs @@ -400,15 +400,13 @@ struct RecordingTransport { filed: StdMutex>, } -static SECRET_TRANSPORT_LOCK: StdMutex<()> = StdMutex::new(()); - struct TransportGuard { _lock: std::sync::MutexGuard<'static, ()>, } impl TransportGuard { fn install(transport: Option>) -> Self { - let guard = SECRET_TRANSPORT_LOCK + let guard = crate::secret::consent::TEST_TRANSPORT_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); crate::secret::consent::install_transport(transport); @@ -788,6 +786,7 @@ fn install_cards(runtime: &tokio::runtime::Runtime, cards: Arc) -> TransportGuard::install(Some(Arc::new(SubcSecretConsent { transport: cards, runtime: runtime.handle().clone(), + cards: Default::default(), }))) } @@ -811,7 +810,11 @@ fn a_refiled_secret_launch_recovers_the_same_card_and_exact_statement() { assert_eq!(cards.cards.lock().unwrap().len(), 1); assert_eq!(wire.calls.lock().unwrap().len(), 2); let calls = cards.calls.lock().unwrap(); - assert_eq!(calls.len(), 4); + assert_eq!( + calls.len(), + 3, + "resume awaits the existing card without filing it again" + ); let item = phone_item(); let expected = format!( r#"{{"method":"consent.request","params":{{"kind":"secret_use","secret_use":{{"items":[{{"request_id":"{}","statement_b64url":"{}"}}]}}}}}}"#, diff --git a/crates/aft/src/url_fetch.rs b/crates/aft/src/url_fetch.rs index 242b826978..aaff9451f0 100644 --- a/crates/aft/src/url_fetch.rs +++ b/crates/aft/src/url_fetch.rs @@ -22,7 +22,8 @@ use reqwest::header::{ use reqwest::redirect::Policy; use secret_rules::audit::{AuditReason, AuditRecordV1, Decision}; use secret_rules::location::{ - evaluate_request_with_connect, LabelOutcome, LabelRule, Location, RequestView, + evaluate_request_with_connect, location_kind_token, LabelOutcome, LabelRule, Location, + RequestView, }; use secret_rules::material::{check_admissible, encode_for_location}; use secret_rules::scrub::{MatcherSet, ScrubError, Scrubber}; @@ -1417,14 +1418,14 @@ fn secret_hop( .map(|host| ConnectAuthority { host, port }) }); let Some(authority) = authority else { - emit_hop_audit(secrets, held, hop, None, |_| { + emit_hop_audit(secrets, held, hop, None, &[], |_| { (Decision::Literal, AuditReason::OutOfScope) }) .map_err(|_| secret_fetch_refused("audit_unavailable", logical))?; return Ok(None); }; let refused = |held: &HeldKeys, reason: &str| { - let _ = emit_hop_audit(secrets, held, hop, Some(&authority), |_| { + let _ = emit_hop_audit(secrets, held, hop, Some(&authority), &[], |_| { (Decision::Refused, AuditReason::Refused) }); secret_fetch_refused(reason, logical) @@ -1441,26 +1442,28 @@ fn secret_hop( path: String::from_utf8(normalized.wire).map_err(|_| refused(held, "malformed_path"))?, }; for label in &secrets.labels { - let mut https_port = authority.port == 443; + let https_port = authority.port == 443; #[cfg(test)] - { - https_port |= secrets.test_https_port == Some(authority.port); - } + let https_port = https_port || secrets.test_https_port == Some(authority.port); if !https_port || !present.contains(label) { continue; } let answer = match crate::subc::use_current_secret_label(secrets.vault, label, &addressed) { crate::subc::LaunchDecision::Proceed(answer) => answer, crate::subc::LaunchDecision::Pending(pending) => { + let _ = refused(held, "missing_material"); return Err(UrlFetchError::secret( crate::secret::SECRET_ACCESS_PENDING, pending.message, - )) + )); } crate::subc::LaunchDecision::Refuse(crate::secret::LabelOutcome::Refuse { code, message, - }) => return Err(UrlFetchError::secret(code, message)), + }) => { + let _ = refused(held, "missing_material"); + return Err(UrlFetchError::secret(code, message)); + } _ => return Err(refused(held, "missing_material")), }; if let crate::secret::UseAnswer::Released(released) = answer { @@ -1486,7 +1489,7 @@ fn secret_hop( } } // Plan with a stand-in key first: the rules decide where a key would go, - // and refuse an ambiguous request, before the vault releases anything. + // and refuse an ambiguous request before copying released bytes into a URL. let planning: Vec<(String, Vec)> = rules .iter() .map(|rule| (rule.label.clone(), SECRET_PLANNING_KEY.to_vec())) @@ -1530,7 +1533,7 @@ fn secret_hop( } }; if edits.is_empty() { - emit_hop_audit(secrets, held, hop, Some(&authority), decide) + emit_hop_audit(secrets, held, hop, Some(&authority), &rules, decide) .map_err(|_| secret_fetch_refused("audit_unavailable", logical))?; return Ok(None); } @@ -1559,7 +1562,7 @@ fn secret_hop( let mut outbound = current.clone(); outbound.set_path(wire_path); outbound.set_query(Some(substituted)); - emit_hop_audit(secrets, held, hop, Some(&authority), decide) + emit_hop_audit(secrets, held, hop, Some(&authority), &rules, decide) .map_err(|_| secret_fetch_refused("audit_unavailable", logical))?; Ok(Some(outbound)) } @@ -1571,6 +1574,7 @@ fn emit_hop_audit( held: &HeldKeys, hop: usize, authority: Option<&ConnectAuthority>, + rules: &[LabelRule], decide: impl Fn(&str) -> (Decision, AuditReason), ) -> Result<(), ()> { let origin = format!("{}#{hop}", secrets.call_id); @@ -1596,6 +1600,10 @@ fn emit_hop_audit( record.host = Some(authority.host.clone()); record.port = Some(authority.port); } + if let Some(rule) = rules.iter().find(|rule| &rule.label == label) { + record.path_prefix = Some(rule.path_prefix.clone()); + record.location_kind = Some(location_kind_token(&rule.location).to_owned()); + } if record.to_json_line_with_keys(&keys).is_err() || secrets.vault.audit(&record).is_err() { failed = true; } From bda7317592409b426c7dcdfc982d9980170c86be Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:45:17 +0100 Subject: [PATCH 039/163] mason: align request audit fields and explain vault route boundaries Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 16aeee0bf251b2d6ea8f0ad2131bcc2fe6efe3d5) (cherry picked from commit 70b387bfa609cf29340c85b2d6bcb539605140f3) --- crates/aft/src/secret/transport.rs | 5 +++-- crates/aft/src/secret_proxy/request.rs | 4 ++-- crates/aft/src/secret_proxy/vault_tests.rs | 3 ++- crates/aft/src/url_fetch.rs | 8 ++++++-- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/crates/aft/src/secret/transport.rs b/crates/aft/src/secret/transport.rs index 6016c5ffe6..47bb2a3e28 100644 --- a/crates/aft/src/secret/transport.rs +++ b/crates/aft/src/secret/transport.rs @@ -34,8 +34,9 @@ pub(crate) fn for_current_command() -> Option { ) } -/// A request-addressed vault transport. Construct it at call -/// time: a standalone process has no module identity and must not open as direct. +/// A request-addressed vault transport. Construct it at call time: a standalone +/// process has no attested AFT module identity. Never open its vault route under +/// the daemon's unprivileged `direct` principal instead. /// One command owns one consumer connection; failed connections are discarded so /// a later request can reconnect after the daemon returns. #[derive(Clone)] diff --git a/crates/aft/src/secret_proxy/request.rs b/crates/aft/src/secret_proxy/request.rs index 56c8391ff8..3990a3f9c4 100644 --- a/crates/aft/src/secret_proxy/request.rs +++ b/crates/aft/src/secret_proxy/request.rs @@ -139,8 +139,8 @@ async fn prepare_and_send( } } } - // A JSON deposit is not known until the addressed release. Buffer bounded - // JSON requests so their placeholders can be discovered before asking. + // The vault names the substitution location only when it releases a key. + // Buffer bounded JSON bodies to discover placeholders before asking it. let has_json = parts .headers .get("content-type") diff --git a/crates/aft/src/secret_proxy/vault_tests.rs b/crates/aft/src/secret_proxy/vault_tests.rs index 0a26213468..a43d20c340 100644 --- a/crates/aft/src/secret_proxy/vault_tests.rs +++ b/crates/aft/src/secret_proxy/vault_tests.rs @@ -1,4 +1,5 @@ -//! Real proxy requests backed by the attested consumer route and a phone double. +//! Real proxy requests using routes authenticated as AFT's module identity. +//! A consent transport double records phone cards and supplies allow or decline. use super::*; use crate::secret::transport::fake::{FakeClaustrum, Record, NONCE}; diff --git a/crates/aft/src/url_fetch.rs b/crates/aft/src/url_fetch.rs index aaff9451f0..a933a7bdc8 100644 --- a/crates/aft/src/url_fetch.rs +++ b/crates/aft/src/url_fetch.rs @@ -1601,8 +1601,12 @@ fn emit_hop_audit( record.port = Some(authority.port); } if let Some(rule) = rules.iter().find(|rule| &rule.label == label) { - record.path_prefix = Some(rule.path_prefix.clone()); - record.location_kind = Some(location_kind_token(&rule.location).to_owned()); + record = record.with_rule( + &rule.host, + rule.port, + &rule.path_prefix, + location_kind_token(&rule.location), + ); } if record.to_json_line_with_keys(&keys).is_err() || secrets.vault.audit(&record).is_err() { failed = true; From 4f1c06aaf10afee9f08fa4c34c4917ebec175606 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:59:22 +0100 Subject: [PATCH 040/163] mason: exercise live route requests and guard concurrent status notices Use an explicit test principal instead of racing process-wide module identity. Keep one SDK consumer bound to one command scope; linked-worker coverage uses its own consumer. Ask only for labels present in requests and validate JSON locations after the vault release. Non-443 tunnels remain opaque; unknown header labels are refused before any request is sent because launch no longer knows a vault host/path. Add a native bash stdout/stderr and bash_status leak regression through the real route. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 8fe3278f921f9ddb12349ce5005934c3e3ec8024) (cherry picked from commit 70344b95d5cea66d36a8652429e695ff39c20914) --- .../bash_background/secrets_audit_tests.rs | 120 +++++++++++++++++- crates/aft/src/secret/requests.rs | 30 +++-- crates/aft/src/secret/tests.rs | 1 + crates/aft/src/secret/transport.rs | 24 ++++ crates/aft/src/secret/transport_tests.rs | 11 +- crates/aft/src/secret_proxy/mod.rs | 2 +- crates/aft/src/secret_proxy/tests.rs | 41 +++--- crates/aft/src/secret_proxy/vault_tests.rs | 25 ++-- 8 files changed, 211 insertions(+), 43 deletions(-) diff --git a/crates/aft/src/bash_background/secrets_audit_tests.rs b/crates/aft/src/bash_background/secrets_audit_tests.rs index a14ba62e78..78b3d8e5bd 100644 --- a/crates/aft/src/bash_background/secrets_audit_tests.rs +++ b/crates/aft/src/bash_background/secrets_audit_tests.rs @@ -37,6 +37,7 @@ struct Fixture { server: Option>, _launcher: Box, tasks: Mutex>, + echo_key: Arc, } impl Fixture { fn new() -> Self { @@ -90,6 +91,8 @@ impl Fixture { let seen = Arc::new(Mutex::new(Vec::new())); let stop = Arc::new(AtomicBool::new(false)); let (seen_server, stopping) = (seen.clone(), stop.clone()); + let echo_key = Arc::new(AtomicBool::new(false)); + let echo_response = echo_key.clone(); let server = std::thread::spawn(move || { while !stopping.load(Ordering::SeqCst) { let socket = match listener.accept() { @@ -120,9 +123,13 @@ impl Fixture { .lock() .unwrap() .push(String::from_utf8(bytes).unwrap()); - let _ = stream.write_all( - b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", - ); + let body: &[u8] = if echo_response.load(Ordering::SeqCst) { + KEY + } else { + b"ok" + }; + let _ = stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes()); + let _ = stream.write_all(body); let _ = stream.flush(); break; } @@ -140,6 +147,7 @@ impl Fixture { server: Some(server), _launcher: Box::new(launcher), tasks: Mutex::new(Vec::new()), + echo_key, } } fn curl(&self) -> String { @@ -152,6 +160,15 @@ impl Fixture { serde_json::from_value(json!({"owner":{"kind":"reserved","module_id":"prefrontal-core"},"ref":"audit-run","kind":"ephemeral","scope_epoch":1,"owner_authorized":true,"parent":{"owner":{"kind":"reserved","module_id":"prefrontal-core"},"ref":"secret-bash","scope_epoch":1},"parent_state":"linked"})).unwrap() } fn launch(&self, command: &str, bound: bool, extra: serde_json::Value) -> Response { + self.launch_with_vault(self.vault.clone(), command, bound, extra) + } + fn launch_with_vault( + &self, + vault: Arc, + command: &str, + bound: bool, + extra: serde_json::Value, + ) -> Response { let mut req = request(command); req.params["params"] .as_object_mut() @@ -163,7 +180,7 @@ impl Fixture { crate::commands::bash::handle(&req, &self.ctx) }) }; - with_vault(self.vault.clone(), || { + with_vault(vault, || { if bound { crate::subc::with_secret_bound_identity_for_test( self.root.path(), @@ -633,3 +650,98 @@ fn audit_hardened_daemon_preserves_child_fd_aliases() { ); }); } + +#[test] +fn audit_live_vault_key_never_reaches_task_logs_previews_or_status() { + isolated( + "audit_live_vault_key_never_reaches_task_logs_previews_or_status", + || { + use crate::secret::transport::fake::{FakeClaustrum, Record, NONCE}; + use crate::secret::{ApiKeyTarget, SubcSecretVault, UseAnswer}; + let f = Fixture::new(); + f.echo_key.store(true, Ordering::SeqCst); + let daemon = FakeClaustrum::start(); + daemon.state.lock().unwrap().record = Some(Record { + name: "test".into(), + host: "localhost".into(), + path: "/api".into(), + location: "authorization_bearer".into(), + key: KEY.to_vec(), + }); + let transport = crate::secret::transport::SubcSecretUseTransport::for_test( + daemon.path.clone(), + subc_protocol::BindIdentity::new("live-secret-bash", "secret-bash"), + Duration::from_secs(2), + subc_client_rs::ConsumerIdentity { + module_id: "aft".into(), + launch_nonce: NONCE.into(), + }, + ); + let vault = Arc::new(SubcSecretVault::new(Arc::new(transport))); + let scope = crate::secret::SecretScope::from_stamp(&stamp("worker")); + let target = ApiKeyTarget { + scheme: "https".into(), + host: "localhost".into(), + port: Some(443), + path: "/api".into(), + }; + let UseAnswer::ApprovalRequired(pending) = vault.use_secret(&scope, "test", &target) + else { + panic!("prime the phone approval"); + }; + daemon + .state + .lock() + .unwrap() + .approve(&pending.statement_b64url); + let command = format!("{}; printf '\\n'; {} >&2", f.curl(), f.curl()); + let response = f.launch_with_vault(vault, &command, false, json!({})); + let id = f.task(&response); + f.wait_terminal(&id); + let snapshot = f + .ctx + .bash_background() + .status( + &id, + "secret-bash", + Some(f.root.path()), + Some(f._storage.path()), + 8192, + ) + .unwrap(); + assert_eq!(snapshot.exit_code, Some(0), "{}", f.diagnostics()); + let key = std::str::from_utf8(KEY).unwrap(); + let status_request: RawRequest = serde_json::from_value(json!({"id":"live-vault-status", "command":"bash_status", "session_id":"secret-bash", "params":{"task_id":id}})).unwrap(); + let status_reply = crate::commands::bash_status::handle(&status_request, &f.ctx); + assert!(status_reply.success, "{status_reply:?}"); + for text in [ + serde_json::to_string(&response).unwrap(), + snapshot.output_preview.clone(), + serde_json::to_string(&snapshot).unwrap(), + f.diagnostics(), + serde_json::to_string(&status_reply).unwrap(), + ] { + assert!( + !text.contains(key), + "released material escaped into task observability" + ); + } + assert!(snapshot.output_preview.contains("{{ck-secret:test}}")); + assert_eq!(f.seen.lock().unwrap().len(), 2); + assert!( + f.seen + .lock() + .unwrap() + .iter() + .all(|request| request.contains(key)), + "the vault key actually reached the approved upstream" + ); + assert_eq!( + daemon.state.lock().unwrap().calls, + 3, + "priming ask and one release per real request" + ); + assert_eq!(daemon.state.lock().unwrap().connections, 1); + }, + ); +} diff --git a/crates/aft/src/secret/requests.rs b/crates/aft/src/secret/requests.rs index bce993036e..c342bf19bc 100644 --- a/crates/aft/src/secret/requests.rs +++ b/crates/aft/src/secret/requests.rs @@ -43,7 +43,8 @@ pub(crate) struct RequestSecrets { scope: SecretScope, context: Option<(std::path::PathBuf, String)>, slots: Mutex>>, - status: Mutex>>, + status: Mutex)>>, + sequence: AtomicU64, clock: Arc, active: Arc, } @@ -60,6 +61,7 @@ impl RequestSecrets { context, slots: Mutex::default(), status: Mutex::default(), + sequence: AtomicU64::new(1), clock: Arc::new(WallClock), active: Arc::default(), } @@ -101,7 +103,7 @@ impl RequestSecrets { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .values() - .flatten() + .flat_map(|(_, lines)| lines) .cloned() .collect() } @@ -131,14 +133,21 @@ impl RequestSecrets { } self.active.fetch_add(1, Ordering::AcqRel); let _active = Active(self.active.clone()); + let sequence = self.sequence.fetch_add(1, Ordering::Relaxed); let result = tokio::select! { - result = self.use_serial(label, target) => result, + result = self.use_serial(label, target, sequence) => result, () = self.clock.deadline() => Err(()), }; - self.status + let mut status = self + .status .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(label); + .unwrap_or_else(std::sync::PoisonError::into_inner); + if status + .get(label) + .is_some_and(|(owner, _)| *owner == sequence) + { + status.remove(label); + } result } @@ -156,7 +165,12 @@ impl RequestSecrets { receive.await.unwrap_or(UseAnswer::Unavailable) } - async fn use_serial(&self, label: &str, target: &ApiKeyTarget) -> Result { + async fn use_serial( + &self, + label: &str, + target: &ApiKeyTarget, + sequence: u64, + ) -> Result { let slot = self.slot(label); let mut history = slot.history.lock().await; if history @@ -197,7 +211,7 @@ impl RequestSecrets { self.status .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert(label.into(), lines); + .insert(label.into(), (sequence, lines)); history.filed = Some((pending.request_id.clone(), pending.expires_at_ms)); slot.expires.store(pending.expires_at_ms, Ordering::Release); let (send, receive) = tokio::sync::oneshot::channel(); diff --git a/crates/aft/src/secret/tests.rs b/crates/aft/src/secret/tests.rs index f98979d77c..b0823e308d 100644 --- a/crates/aft/src/secret/tests.rs +++ b/crates/aft/src/secret/tests.rs @@ -177,6 +177,7 @@ fn the_fake_counts_addressed_uses_and_rotates_material() { assert_eq!(second.material(), b"second-key-material"); assert_eq!(vault.use_calls(), 2); assert_eq!(vault.material_calls(), 2); + assert_eq!(vault.materials(), vec!["repo", "repo"]); let mut other = target(); other.host = "other.example.com".into(); assert!(matches!( diff --git a/crates/aft/src/secret/transport.rs b/crates/aft/src/secret/transport.rs index 47bb2a3e28..81d6532350 100644 --- a/crates/aft/src/secret/transport.rs +++ b/crates/aft/src/secret/transport.rs @@ -45,6 +45,8 @@ pub struct SubcSecretUseTransport { identity: BindIdentity, timeout: Duration, worker: Arc>>, + #[cfg(test)] + test_principal: Option, } struct Work { @@ -74,9 +76,27 @@ impl SubcSecretUseTransport { identity, timeout, worker: Arc::default(), + #[cfg(test)] + test_principal: None, }) } + #[cfg(test)] + pub(crate) fn for_test( + connection_file: PathBuf, + identity: BindIdentity, + timeout: Duration, + principal: ConsumerIdentity, + ) -> Self { + Self { + connection_file, + identity, + timeout, + worker: Arc::default(), + test_principal: Some(principal), + } + } + async fn send( config: &ConnectionConfig, consumer: &subc_client_rs::SubcConsumer, @@ -217,6 +237,10 @@ fn aft_identity() -> Option { impl SecretUseTransport for SubcSecretUseTransport { fn secret_use(&self, params: Value) -> Result { + #[cfg(test)] + if let Some(principal) = &self.test_principal { + return self.dispatch(params, principal.clone()); + } let principal = aft_identity().ok_or_else(|| UNAVAILABLE.to_owned())?; self.dispatch(params, principal) } diff --git a/crates/aft/src/secret/transport_tests.rs b/crates/aft/src/secret/transport_tests.rs index b304354013..f05e136a97 100644 --- a/crates/aft/src/secret/transport_tests.rs +++ b/crates/aft/src/secret/transport_tests.rs @@ -15,6 +15,7 @@ fn transport(daemon: &FakeClaustrum) -> SubcSecretUseTransport { identity: BindIdentity::new("aft-secret-use", "secret-call"), timeout: Duration::from_secs(2), worker: Arc::default(), + test_principal: None, } } @@ -119,16 +120,20 @@ fn secret_use_route_maps_all_sender_refusals_without_releasing_material() { #[test] fn secret_use_fake_grant_lookup_is_independent_of_requested_coverage_and_covers_linked_workers() { let daemon = FakeClaustrum::start(); - let transport = transport(&daemon); + let head_transport = transport(&daemon); let mut head = golden(0); head["coverage"] = json!("agent_and_workers"); - let pending = call(&transport, head); + let pending = call(&head_transport, head); daemon .state .lock() .unwrap() .approve(pending["result"]["statement_b64url"].as_str().unwrap()); - assert_eq!(call(&transport, golden(1)), json!({"result":golden(3)})); + assert_eq!( + call(&head_transport, golden(1)), + json!({"result":golden(3)}) + ); + let transport = transport(&daemon); let mut worker = golden(0); worker["stamp"] = golden(2)["stamp"].clone(); assert_eq!( diff --git a/crates/aft/src/secret_proxy/mod.rs b/crates/aft/src/secret_proxy/mod.rs index 4107ecee3f..e4cb511e10 100644 --- a/crates/aft/src/secret_proxy/mod.rs +++ b/crates/aft/src/secret_proxy/mod.rs @@ -98,7 +98,7 @@ pub struct ProxyConfig { #[cfg(test)] pub test_https_port: Option, #[cfg(test)] - pub approval_clock: Option>, + pub(crate) approval_clock: Option>, pub origin: String, } diff --git a/crates/aft/src/secret_proxy/tests.rs b/crates/aft/src/secret_proxy/tests.rs index 1d46fe5c66..097c9b0f16 100644 --- a/crates/aft/src/secret_proxy/tests.rs +++ b/crates/aft/src/secret_proxy/tests.rs @@ -787,7 +787,11 @@ async fn proxy_h1_h2_locations_keepalive_authority_and_scanned_set() { let response = client.command(json!({"op":"request", "authority":"other.invalid", "path":"/approved", "headers":{"authorization":format!("Bearer {MARKER}")}})); assert_eq!(response["status"], 403, "{response}"); } - assert_eq!(vault.calls.load(Ordering::SeqCst), 16); + assert_eq!( + vault.calls.load(Ordering::SeqCst), + 10, + "only labels present in each request are asked" + ); proxy.stop(); assert!(proxy.state.patterns.lock().is_empty()); } @@ -877,9 +881,10 @@ async fn proxy_literal_responses_only_gain_matchers_after_material() { ); assert!(!after.to_string().contains(std::str::from_utf8(K1).unwrap())); assert_eq!( - server.seen.lock().last().unwrap()["headers"]["authorization"], - "Bearer {{ck-secret:from-file}}" + after["status"], 403, + "an unscanned header label is refused before dispatch" ); + assert_eq!(server.seen.lock().len(), 2); assert_eq!(vault.calls.load(Ordering::SeqCst), 1); } @@ -903,12 +908,16 @@ async fn proxy_atomic_json_and_material_refusals_send_nothing_upstream() { proxy.arm_command(client.process.id()).unwrap(); client.connect(false); for body in [ - r#"{"key":4}"#, + r#"{"key":4,"other":"{{ck-secret:json}}"}"#, r#"{"key":"{{ck-secret:json}}","key":"{{ck-secret:json}}"}"#, ] { assert_eq!(client.request("/approved", json!({"authorization":format!("Bearer {MARKER}"),"content-type":"application/json"}), body)["status"], 403); } - assert_eq!(vault.calls.load(Ordering::SeqCst), 0); + assert_eq!( + vault.calls.load(Ordering::SeqCst), + 4, + "the vault names the location before its structure can be validated" + ); assert!(server.seen.lock().is_empty()); *vault.refused_label.lock() = Some("json".into()); let refused = client.request( @@ -921,7 +930,7 @@ async fn proxy_atomic_json_and_material_refusals_send_nothing_upstream() { refused["body"], "Secret proxy refused the request: json (localhost)\n" ); - assert_eq!(vault.calls.load(Ordering::SeqCst), 2); + assert_eq!(vault.calls.load(Ordering::SeqCst), 6); assert!(server.seen.lock().is_empty()); *vault.refused_label.lock() = None; assert_eq!( @@ -1024,18 +1033,16 @@ async fn proxy_websocket_scrubs_fragments_and_refuses_extensions_and_rsv() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn proxy_non_rule_connect_is_tunnelled_without_interception() { +async fn proxy_non_https443_connect_is_tunnelled_without_interception() { let server = Server::new("echo-request").await; let vault = Vault::new(); - let proxy = SecretProxy::start_with_tls( - config( - vault.clone(), - server.address.port(), - &[("test", Location::Bearer)], - ), - server.client_tls.clone(), - ) - .unwrap(); + let mut cfg = config( + vault.clone(), + server.address.port(), + &[("test", Location::Bearer)], + ); + cfg.test_https_port = None; + let proxy = SecretProxy::start_with_tls(cfg, server.client_tls.clone()).unwrap(); let mut client = Client::spawn(&proxy, &server); proxy.arm_command(client.process.id()).unwrap(); assert_eq!( @@ -1310,7 +1317,7 @@ async fn proxy_unavailable_release_refuses_and_tunnel_closes_on_stop() { )["status"], 403 ); - assert_eq!(vault.calls.load(Ordering::SeqCst), 0); + assert_eq!(vault.calls.load(Ordering::SeqCst), 1); assert!(server.seen.lock().is_empty()); assert_eq!( client.command(json!({"op":"connect","authority":server.address.to_string()}))["ok"], diff --git a/crates/aft/src/secret_proxy/vault_tests.rs b/crates/aft/src/secret_proxy/vault_tests.rs index a43d20c340..f046752475 100644 --- a/crates/aft/src/secret_proxy/vault_tests.rs +++ b/crates/aft/src/secret_proxy/vault_tests.rs @@ -166,12 +166,15 @@ fn live_config(daemon: &FakeClaustrum, port: u16, clock: Arc) -> ProxyCon "live-vault-{}", SEQUENCE.fetch_add(1, Ordering::SeqCst) )); - let transport = crate::secret::transport::SubcSecretUseTransport::new( + let transport = crate::secret::transport::SubcSecretUseTransport::for_test( daemon.path.clone(), subc_protocol::BindIdentity::new("test", "session"), Duration::from_secs(2), - ) - .unwrap(); + subc_client_rs::ConsumerIdentity { + module_id: "aft".into(), + launch_nonce: NONCE.into(), + }, + ); ProxyConfig { vault: Arc::new(SubcSecretVault::new(Arc::new(transport))), authorization_scope: auth, @@ -204,7 +207,6 @@ fn request_in_thread( #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn live_vault_release_injects_only_the_deposit_location_and_every_use_is_audited() { - let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); let daemon = FakeClaustrum::start(); daemon.state.lock().unwrap().record = Some(record("authorization_bearer")); let phone = PhoneGuard::install(); @@ -256,7 +258,6 @@ async fn live_vault_release_injects_only_the_deposit_location_and_every_use_is_a #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn live_vault_decline_and_expiry_answer_403_without_sending_or_retrying() { - let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); for decision in [ConsentOutcome::Declined, ConsentOutcome::Expired] { let daemon = FakeClaustrum::start(); daemon.state.lock().unwrap().record = Some(record("authorization_bearer")); @@ -304,7 +305,6 @@ async fn live_vault_decline_and_expiry_answer_403_without_sending_or_retrying() #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn live_vault_injected_ten_minute_deadline_answers_403_locally() { - let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); let daemon = FakeClaustrum::start(); daemon.state.lock().unwrap().record = Some(record("authorization_bearer")); let phone = PhoneGuard::install(); @@ -315,11 +315,20 @@ async fn live_vault_injected_ten_minute_deadline_answers_403_locally() { server.client_tls.clone(), ) .unwrap(); + proxy + .state + .test_idle_timeout_ms + .store(100, Ordering::Release); let mut client = Client::spawn(&proxy, &server); proxy.arm_command(client.process.id()).unwrap(); client.connect(false); let worker = request_in_thread(client, false); phone.phone.card(1).await; + tokio::time::sleep(Duration::from_millis(250)).await; + assert!( + !worker.is_finished(), + "a held approval outlives the socket idle timeout" + ); clock.advance(599_999); assert!( !worker.is_finished(), @@ -339,7 +348,6 @@ async fn live_vault_injected_ten_minute_deadline_answers_403_locally() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn live_vault_concurrent_requests_share_one_card_and_once_queues_cards_serially() { - let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); for once in [false, true] { let daemon = FakeClaustrum::start(); { @@ -388,7 +396,6 @@ async fn live_vault_concurrent_requests_share_one_card_and_once_queues_cards_ser #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn live_vault_wrong_host_is_literal_and_never_files_a_card() { - let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); let daemon = FakeClaustrum::start(); let mut other = record("authorization_bearer"); other.host = "other.test".into(); @@ -421,7 +428,6 @@ async fn live_vault_wrong_host_is_literal_and_never_files_a_card() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn live_vault_redirect_to_another_host_asks_again_and_sends_no_key() { - let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); let daemon = FakeClaustrum::start(); daemon.state.lock().unwrap().record = Some(record("authorization_bearer")); let phone = PhoneGuard::install(); @@ -463,7 +469,6 @@ async fn live_vault_redirect_to_another_host_asks_again_and_sends_no_key() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn live_vault_fetch_uses_the_concrete_target_and_holds_for_the_phone() { - let _nonce = crate::launch_nonce::install_for_tests(Some(NONCE.into())); let daemon = FakeClaustrum::start(); daemon.state.lock().unwrap().record = Some(record("query:key")); let phone = PhoneGuard::install(); From dfa539304ac628709d487c0185059da4ee062dc3 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:05:49 +0100 Subject: [PATCH 041/163] mason: correct cross-platform proxy types and ephemeral upstream fixtures Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit f5cd97072dde685afad0740330c5065c6443abc4) (cherry picked from commit 86f6351d3fba4e1bdabf533f1d81a8765e77f214) --- crates/aft/src/commands/url_resolution.rs | 6 +----- crates/aft/src/secret/mod.rs | 1 - crates/aft/src/secret_proxy/tests.rs | 9 +-------- crates/aft/src/secret_proxy/vault_tests.rs | 6 +----- crates/aft/src/subc/secret_consent.rs | 5 +++-- 5 files changed, 6 insertions(+), 21 deletions(-) diff --git a/crates/aft/src/commands/url_resolution.rs b/crates/aft/src/commands/url_resolution.rs index 3675df69a9..a709aadeee 100644 --- a/crates/aft/src/commands/url_resolution.rs +++ b/crates/aft/src/commands/url_resolution.rs @@ -186,11 +186,7 @@ fn resolve_secret_fetch( authorization, labels: labels.to_vec(), #[cfg(test)] - test_https_port: if options.allow_private { - url::Url::parse(file).ok().and_then(|url| url.port()) - } else { - None - }, + test_https_port: url::Url::parse(file).ok().and_then(|url| url.port()), call_id: req.id.clone(), }; let document = crate::url_fetch::fetch_url_with_secrets(file, &document_dir, options, &secrets) diff --git a/crates/aft/src/secret/mod.rs b/crates/aft/src/secret/mod.rs index 4026cc4983..21a30b7573 100644 --- a/crates/aft/src/secret/mod.rs +++ b/crates/aft/src/secret/mod.rs @@ -14,7 +14,6 @@ pub mod consent; pub mod eligibility; pub mod outcome; -#[cfg(unix)] pub(crate) mod requests; pub mod scope; pub mod transport; diff --git a/crates/aft/src/secret_proxy/tests.rs b/crates/aft/src/secret_proxy/tests.rs index 097c9b0f16..0580a4e2ac 100644 --- a/crates/aft/src/secret_proxy/tests.rs +++ b/crates/aft/src/secret_proxy/tests.rs @@ -1323,14 +1323,7 @@ async fn proxy_unavailable_release_refuses_and_tunnel_closes_on_stop() { client.command(json!({"op":"connect","authority":server.address.to_string()}))["ok"], true ); - assert_eq!( - client.request( - "/approved", - json!({"authorization":format!("Bearer {MARKER}")}), - "" - )["status"], - 200 - ); + assert_eq!(client.command(json!({"op":"request", "authority":server.address.to_string(), "path":"/approved", "headers":{"authorization":format!("Bearer {MARKER}")}}))["status"], 200); proxy.process_tree_ended(); assert_post_stop_connect_is_not_served(&proxy, server.address).await; assert_eq!(client.request("/approved", json!({}), "")["error"], true); diff --git a/crates/aft/src/secret_proxy/vault_tests.rs b/crates/aft/src/secret_proxy/vault_tests.rs index f046752475..386a147309 100644 --- a/crates/aft/src/secret_proxy/vault_tests.rs +++ b/crates/aft/src/secret_proxy/vault_tests.rs @@ -453,11 +453,7 @@ async fn live_vault_redirect_to_another_host_asks_again_and_sends_no_key() { )["ok"], true ); - let second = client.request( - location.path(), - json!({"authorization":format!("Bearer {MARKER}")}), - "", - ); + let second = client.command(json!({"op":"request", "authority":format!("127.0.0.1:{}", server.address.port()), "path":location.path(), "headers":{"authorization":format!("Bearer {MARKER}")}})); assert_eq!(second["status"], 200); assert_eq!( server.seen.lock()[1]["headers"]["authorization"], diff --git a/crates/aft/src/subc/secret_consent.rs b/crates/aft/src/subc/secret_consent.rs index 1709af68bc..81d9186d89 100644 --- a/crates/aft/src/subc/secret_consent.rs +++ b/crates/aft/src/subc/secret_consent.rs @@ -41,8 +41,9 @@ use subc_protocol::scope::ScopeStamp; const SECRET_CONSENT_TIMEOUT: Duration = Duration::from_secs(10); -/// Use the same scoped daemon connection and reserved AFT filer as permission -/// asks, retaining the card id so a held call never files it twice. +/// Use the daemon connection that files filesystem-access questions with +/// Cingulate as its reserved AFT module principal. Retain each secret card's id +/// so a held call waits for the existing card instead of filing another. pub(super) fn install_subc_transport(transport: Arc) { crate::secret::consent::install_transport(Some(Arc::new(SubcSecretConsent { transport, From 705d2d9384e387dddea0d8feac4a0356405dd03f Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:15:36 +0100 Subject: [PATCH 042/163] mason: bind redirect fixture to a verified second DNS host A release supplies its location, so a present placeholder must reach secret.use before location planning can decide to leave it literal. Update that old material-only assertion explicitly; placeholder-free redirect hops still do not ask. Fix Windows-only unused helper warnings without changing supported platforms. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 49fd2deb6fb2d692c8e9ac9c86869d5a6f9687bc) (cherry picked from commit 3893d1ac909707a3236c495b678d7531be4cab83) --- crates/aft/src/bash_background/secrets_tests.rs | 8 +++++--- crates/aft/src/commands/url_resolution.rs | 13 ++++++++----- crates/aft/src/secret/requests.rs | 1 + crates/aft/src/secret/transport.rs | 2 +- crates/aft/src/secret_proxy/mod.rs | 5 +++++ crates/aft/src/secret_proxy/request.rs | 14 ++++++++++---- crates/aft/src/secret_proxy/tests.rs | 9 +++++++-- crates/aft/src/secret_proxy/vault_tests.rs | 11 ++++++++--- 8 files changed, 45 insertions(+), 18 deletions(-) diff --git a/crates/aft/src/bash_background/secrets_tests.rs b/crates/aft/src/bash_background/secrets_tests.rs index e147f7dceb..8beaa6aa80 100644 --- a/crates/aft/src/bash_background/secrets_tests.rs +++ b/crates/aft/src/bash_background/secrets_tests.rs @@ -11,6 +11,7 @@ mod audit_followups; struct Vault { resolves: AtomicUsize, materials: AtomicUsize, + #[cfg(unix)] port: u16, audits: Mutex>, resolve_kinds: Mutex>, @@ -19,13 +20,14 @@ struct Vault { } impl Vault { - fn new(port: u16) -> Arc { + fn new(_port: u16) -> Arc { #[cfg(unix)] - TEST_HTTPS_PORT.with(|slot| slot.set((port != 0).then_some(port))); + TEST_HTTPS_PORT.with(|slot| slot.set((_port != 0).then_some(_port))); Arc::new(Self { resolves: AtomicUsize::new(0), materials: AtomicUsize::new(0), - port, + #[cfg(unix)] + port: _port, audits: Mutex::new(Vec::new()), resolve_kinds: Mutex::new(Vec::new()), material_kinds: Mutex::new(Vec::new()), diff --git a/crates/aft/src/commands/url_resolution.rs b/crates/aft/src/commands/url_resolution.rs index a709aadeee..7699088c50 100644 --- a/crates/aft/src/commands/url_resolution.rs +++ b/crates/aft/src/commands/url_resolution.rs @@ -1907,11 +1907,10 @@ mod url_tests { assert_absent(&run, &documents, &[KEY, ENCODED_KEY.as_bytes()]); } - /// `material` is asked only for a hop that substitutes: not for a - /// placeholder in another parameter, and not for a same-prefix - /// redirect hop that carries no placeholder. + /// A request must ask the vault before it knows the key's location. + /// A redirect without a placeholder has nothing to ask about. #[test] - fn material_is_requested_only_for_a_hop_that_substitutes() { + fn present_labels_ask_before_location_planning_but_empty_redirects_do_not() { let fixture = fixture(Default::default()); let server = serve(&fixture.tls, |target| { if target.starts_with("/v1/first.md") { @@ -1933,7 +1932,11 @@ mod url_tests { no_extra, ); assert!(run.response.success, "{:?}", run.response.data); - assert_eq!(vault.material_calls(), 0, "nothing was substituted"); + assert_eq!( + vault.material_calls(), + 1, + "release names the location; a placeholder in another parameter stays literal" + ); let redirected = format!("https://api.secret.test:{port}/v1/first.md?key={PLACEHOLDER}"); diff --git a/crates/aft/src/secret/requests.rs b/crates/aft/src/secret/requests.rs index c342bf19bc..bf04142bab 100644 --- a/crates/aft/src/secret/requests.rs +++ b/crates/aft/src/secret/requests.rs @@ -98,6 +98,7 @@ impl RequestSecrets { .clone() } + #[cfg(unix)] pub(crate) fn status(&self) -> Vec { self.status .lock() diff --git a/crates/aft/src/secret/transport.rs b/crates/aft/src/secret/transport.rs index 81d6532350..8fc4cefeeb 100644 --- a/crates/aft/src/secret/transport.rs +++ b/crates/aft/src/secret/transport.rs @@ -81,7 +81,7 @@ impl SubcSecretUseTransport { }) } - #[cfg(test)] + #[cfg(all(test, unix))] pub(crate) fn for_test( connection_file: PathBuf, identity: BindIdentity, diff --git a/crates/aft/src/secret_proxy/mod.rs b/crates/aft/src/secret_proxy/mod.rs index e4cb511e10..836b314bbc 100644 --- a/crates/aft/src/secret_proxy/mod.rs +++ b/crates/aft/src/secret_proxy/mod.rs @@ -193,6 +193,8 @@ pub(crate) struct State { test_identity_generation: std::sync::atomic::AtomicU64, #[cfg(test)] test_idle_timeout_ms: std::sync::atomic::AtomicU64, + #[cfg(test)] + test_upstream: Mutex>, } impl State { @@ -409,6 +411,8 @@ impl SecretProxy { test_identity_generation: std::sync::atomic::AtomicU64::new(0), #[cfg(test)] test_idle_timeout_ms: std::sync::atomic::AtomicU64::new(0), + #[cfg(test)] + test_upstream: Mutex::default(), }); let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) .map_err(|_| ProxyError::Initialization)?; @@ -447,6 +451,7 @@ impl SecretProxy { self.address } + #[cfg(unix)] pub(crate) fn approval_status(&self) -> Vec { self.state.approvals.status() } diff --git a/crates/aft/src/secret_proxy/request.rs b/crates/aft/src/secret_proxy/request.rs index 3990a3f9c4..be437e1355 100644 --- a/crates/aft/src/secret_proxy/request.rs +++ b/crates/aft/src/secret_proxy/request.rs @@ -439,10 +439,16 @@ async fn connect_upstream( websocket: bool, ) -> Result { let fail = || String::new(); - let socket = tokio::time::timeout( - state.idle_timeout(), - tokio::net::TcpStream::connect((super::network_host(&authority.host), authority.port)), - ) + let socket = tokio::time::timeout(state.idle_timeout(), async { + #[cfg(test)] + { + let address = state.test_upstream.lock().get(&authority.host).copied(); + if let Some(address) = address { + return tokio::net::TcpStream::connect(address).await; + } + } + tokio::net::TcpStream::connect((super::network_host(&authority.host), authority.port)).await + }) .await .map_err(|_| fail())? .map_err(|_| fail())?; diff --git a/crates/aft/src/secret_proxy/tests.rs b/crates/aft/src/secret_proxy/tests.rs index 0580a4e2ac..9a38ea49c5 100644 --- a/crates/aft/src/secret_proxy/tests.rs +++ b/crates/aft/src/secret_proxy/tests.rs @@ -165,7 +165,12 @@ impl Server { let names = if mode == "wrong-name" { vec!["wrong.invalid".into()] } else { - vec!["localhost".into(), "127.0.0.1".into(), "::1".into()] + vec![ + "localhost".into(), + "127.0.0.1".into(), + "::1".into(), + "other.test".into(), + ] }; let mut params = CertificateParams::new(names).unwrap(); if mode == "expired" { @@ -288,7 +293,7 @@ impl Server { response.headers_mut().insert( "location", HeaderValue::from_str(&format!( - "https://127.0.0.1:{port}/approved" + "https://other.test:{port}/approved" )) .unwrap(), ); diff --git a/crates/aft/src/secret_proxy/vault_tests.rs b/crates/aft/src/secret_proxy/vault_tests.rs index 386a147309..34ea72400b 100644 --- a/crates/aft/src/secret_proxy/vault_tests.rs +++ b/crates/aft/src/secret_proxy/vault_tests.rs @@ -446,14 +446,19 @@ async fn live_vault_redirect_to_another_host_asks_again_and_sends_no_key() { let (first, mut client) = worker.join().unwrap(); assert_eq!(first["status"], 302); let location = url::Url::parse(first["headers"]["location"].as_str().unwrap()).unwrap(); - assert_eq!(location.host_str(), Some("127.0.0.1")); + assert_eq!(location.host_str(), Some("other.test")); + proxy + .state + .test_upstream + .lock() + .insert("other.test".into(), server.address); assert_eq!( client.command( - json!({"op":"connect", "authority":format!("127.0.0.1:{}", server.address.port())}) + json!({"op":"connect", "authority":format!("other.test:{}", server.address.port())}) )["ok"], true ); - let second = client.command(json!({"op":"request", "authority":format!("127.0.0.1:{}", server.address.port()), "path":location.path(), "headers":{"authorization":format!("Bearer {MARKER}")}})); + let second = client.command(json!({"op":"request", "authority":format!("other.test:{}", server.address.port()), "path":location.path(), "headers":{"authorization":format!("Bearer {MARKER}")}})); assert_eq!(second["status"], 200); assert_eq!( server.seen.lock()[1]["headers"]["authorization"], From 4bd150df3c177f6072c3acfe08f56580eb9855c2 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:38:01 +0100 Subject: [PATCH 043/163] mason: verify held phone status and document live vault release The approved bash launch contract allows an eligible placeholder command to start without consulting an unavailable vault; update the old launch-refusal assertion explicitly. Strengthen the native real-route regression to hold the first HTTP request and check its approved waiting text in bash_status before allowing it. The release note now describes the request-addressed feature proven by the proxy and native bash paths. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 2af82ce64660fa0ef6ad0a60aeb73608fc6cd4c3) (cherry picked from commit 00ec7105ca1f0d2aa7504d95aa770cf49713c1a4) --- .../release-notes/next/live-secret-vault.md | 2 +- .../bash_background/secrets_audit_tests.rs | 68 ++++++++++++++----- .../aft/src/bash_background/secrets_tests.rs | 5 +- crates/aft/src/secret/vault/fake.rs | 25 +++---- 4 files changed, 68 insertions(+), 32 deletions(-) diff --git a/.alfonso/release-notes/next/live-secret-vault.md b/.alfonso/release-notes/next/live-secret-vault.md index a615251f6a..ed1a258547 100644 --- a/.alfonso/release-notes/next/live-secret-vault.md +++ b/.alfonso/release-notes/next/live-secret-vault.md @@ -1 +1 @@ -Added the bounded, daemon-authenticated transport and contract fixtures for live vault secret use. Shell secret placeholders remain unavailable until the request-addressed vault client is connected. +Shell `{{ck-secret: