From da419cc8f488512bd85f5f729811356690c59b53 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 07:26:34 +0100 Subject: [PATCH 001/106] Use local SDK and preserve fork promotion across publication --- Cargo.lock | 9 ++--- Cargo.toml | 3 +- crates/acyclic-engine/src/pipeline.rs | 47 ++++++++++++++++++++++----- tests/acceptance/windows-smoke.sh | 20 ++++++------ 4 files changed, 53 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 426e5bc..d6e40f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,8 +41,7 @@ dependencies = [ [[package]] name = "acyclic-fs" -version = "0.2.0-rc.1" -source = "git+https://github.com/acyclic-labs/sdk.git?rev=22b4e752f46d8f6db6e024b3137fe520ef64bcc1#22b4e752f46d8f6db6e024b3137fe520ef64bcc1" +version = "0.2.0-rc.5" dependencies = [ "acyclic-objects", "acyclic-stream", @@ -73,8 +72,7 @@ dependencies = [ [[package]] name = "acyclic-objects" -version = "1.0.0-rc.1" -source = "git+https://github.com/acyclic-labs/sdk.git?rev=22b4e752f46d8f6db6e024b3137fe520ef64bcc1#22b4e752f46d8f6db6e024b3137fe520ef64bcc1" +version = "1.0.0-rc.4" dependencies = [ "async-trait", "blake3", @@ -107,8 +105,7 @@ dependencies = [ [[package]] name = "acyclic-stream" -version = "1.0.0-rc.3" -source = "git+https://github.com/acyclic-labs/sdk.git?rev=22b4e752f46d8f6db6e024b3137fe520ef64bcc1#22b4e752f46d8f6db6e024b3137fe520ef64bcc1" +version = "1.0.0-rc.7" dependencies = [ "async-trait", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 80afe71..126b8cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ repository = "https://github.com/acyclic-labs/graphcoder-plugin" homepage = "https://acyclic.dev" [workspace.dependencies] -acyclic-fs = { git = "https://github.com/acyclic-labs/sdk.git", rev = "22b4e752f46d8f6db6e024b3137fe520ef64bcc1", features = ["local", "native-watch", "native-mount"] } +acyclic-fs = { path = "../../../sdk/rust/crates/filesystem", features = ["local", "native-watch", "native-mount"] } tokio = { version = "1.48", features = ["rt-multi-thread", "macros"] } # Code-quality guards beyond clippy's defaults. CI runs clippy with @@ -67,4 +67,3 @@ cast_lossless = "warn" # function naming the invariant, so a reviewer can find them all. unsafe_code = "warn" unsafe_op_in_unsafe_fn = "warn" - diff --git a/crates/acyclic-engine/src/pipeline.rs b/crates/acyclic-engine/src/pipeline.rs index 8a24942..b2e6903 100644 --- a/crates/acyclic-engine/src/pipeline.rs +++ b/crates/acyclic-engine/src/pipeline.rs @@ -33,6 +33,15 @@ const MAXIMUM_EXTENT_SPANS: u32 = 65_536; const WATCH_QUEUE: u32 = 65_536; const POLL_CHANGES: u32 = 16_384; +fn fork_moved(base: GenerationId) -> PromoteOutcome { + PromoteOutcome::Conflict { + message: format!( + "the working tree moved past the fork's base ({}); promote in v1 requires an unmoved mainline — rewind to the base or re-fork and re-apply", + crate::generation_hex(base) + ), + } +} + /// Pipeline state reported by `status`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum State { @@ -2021,6 +2030,13 @@ impl Pipeline { self.last_checkpoint_row = Some(safety_row); self.commit_engine().await?; + // The fork may have been cut from an unpublished checkpoint. Publishing + // that same generation advances the authority sequence, so its checkout + // must rebase before committing even though the mainline did not move. + if self.store.checkout.generation_id() != base { + return Ok(fork_moved(base)); + } + let outcome = { let mut guard = shared.lock().await; if !guard.has_pending_mutations() { @@ -2031,6 +2047,17 @@ impl Pipeline { old_tree: None, }); } + let rebased = guard + .rebase_head(0, WorkCounters::UNBOUNDED, &self.cancel) + .await + .map_err(EngineError::fs("fork rebase"))? + .value; + if matches!( + rebased, + acyclic_fs::kernel::RebaseDecision::Conflicted { .. } + ) { + return Ok(fork_moved(base)); + } guard .commit(OperationId::new(), WorkCounters::UNBOUNDED, &self.cancel) .await @@ -2041,14 +2068,7 @@ impl Pipeline { CheckoutCommitOutcome::Committed { generation_id, .. } | CheckoutCommitOutcome::AlreadyCommitted { generation_id, .. } => generation_id, CheckoutCommitOutcome::Conflict { .. } | CheckoutCommitOutcome::Fenced { .. } => { - return Ok(PromoteOutcome::Conflict { - message: format!( - "the working tree moved past the fork's base \ - ({}); promote in v1 requires an unmoved mainline — \ - rewind to the base or re-fork and re-apply", - crate::generation_hex(base) - ), - }); + return Ok(fork_moved(base)); } other => { return Err(EngineError::Fs(format!( @@ -2175,6 +2195,17 @@ impl Pipeline { label: &str, ) -> Result { self.state = State::Rewinding; + self.drain_watcher().await?; + if self.checkpoint_engine().await? != base { + self.state = State::Ready; + return Ok(PromoteOutcome::Conflict { + message: format!( + "the working tree moved past the session's base ({}); Safe Mode in v1 requires an unmoved mainline", + crate::generation_hex(base) + ), + }); + } + self.commit_engine().await?; self.store.checkout = self .store .volume diff --git a/tests/acceptance/windows-smoke.sh b/tests/acceptance/windows-smoke.sh index fe60dad..fe9149f 100644 --- a/tests/acceptance/windows-smoke.sh +++ b/tests/acceptance/windows-smoke.sh @@ -25,9 +25,6 @@ WORK="$(mktemp -d)" REPO="$WORK/repo" cleanup() { "$ACYCLIC" --repo "$REPO" stop >/dev/null 2>&1 || true - # A daemon that outlives the run holds the store open and fails the next one. - powershell -NoProfile -Command \ - "Stop-Process -Name acyclic -Force -ErrorAction SilentlyContinue" >/dev/null 2>&1 || true rm -rf "$WORK" 2>/dev/null || true } trap cleanup EXIT @@ -57,15 +54,18 @@ powershell -NoProfile -ExecutionPolicy Bypass -File "$HERE/windows-pipe-acl.ps1" || fail "the daemon pipe is not owner-only" echo "--- checkpoint and timeline" -# Let the baseline settle before editing. `init` returning does not guarantee -# the baseline predates a write landing microseconds later: the change gets -# folded into checkpoint #1 rather than appearing as a later one, and the diff -# below then has nothing to report. That race is not Windows-specific, and is -# not what this script is here to test. -sleep 3 +# Wait for the observable baseline, not an assumed startup duration. +baseline_ready=false +for _ in {1..100}; do + if "$ACYCLIC" timeline < /dev/null 2>/dev/null | grep -q baseline; then + baseline_ready=true + break + fi + sleep 0.05 +done +[ "$baseline_ready" = true ] || fail "baseline did not become ready" printf 'DIFFERENT AND LONGER CONTENT\n' > src/main.rs printf 'extra\n' > added.txt -sleep 2 "$ACYCLIC" checkpoint < /dev/null > /dev/null || fail "checkpoint failed" "$ACYCLIC" timeline < /dev/null | grep -q baseline || fail "timeline has no baseline" From 86be1dcaf1ee8b39bb3d707d921e4984bb11085c Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 07:41:53 +0100 Subject: [PATCH 002/106] Bound Windows daemon calls with overlapped pipe I/O --- crates/acyclic/Cargo.toml | 2 + crates/acyclic/src/client.rs | 122 +++++++++++++++++++-- crates/acyclic/src/ipc.rs | 203 +++++++++++++++++++++++++++++++++-- docs/windows-verification.md | 10 +- 4 files changed, 311 insertions(+), 26 deletions(-) diff --git a/crates/acyclic/Cargo.toml b/crates/acyclic/Cargo.toml index 285a10b..733543c 100644 --- a/crates/acyclic/Cargo.toml +++ b/crates/acyclic/Cargo.toml @@ -29,9 +29,11 @@ libc = "0.2" # timeout reach the child's own children (`spec_runner::RunGroup`). windows-sys = { version = "0.61", features = [ "Win32_Foundation", + "Win32_Storage_FileSystem", "Win32_Security", "Win32_Security_Authorization", "Win32_System_JobObjects", + "Win32_System_IO", "Win32_System_Memory", "Win32_System_Threading", ] } diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index 79606a6..1113b1c 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -25,8 +25,10 @@ pub enum Spawn { pub struct Client { stream: BufReader, next_id: u64, + call_timeout: Option, } +#[derive(Debug)] pub enum ConnectError { /// No daemon and spawning was not allowed. NoDaemon, @@ -90,14 +92,14 @@ impl Client { Ok(Self { stream: BufReader::new(stream), next_id: 1, + call_timeout: None, }) } /// Bounds how long a single call may wait for its reply. Used by the /// pre-tool hook: an exact boundary is worth milliseconds, not seconds. pub fn set_deadline(&mut self, deadline: std::time::Duration) { - let _ = self.stream.get_ref().set_read_timeout(Some(deadline)); - let _ = self.stream.get_ref().set_write_timeout(Some(deadline)); + self.call_timeout = Some(deadline); } pub fn call(&mut self, op: proto::Op) -> Result { @@ -129,6 +131,7 @@ impl Client { } fn call_inner(&mut self, id: u64, op: proto::Op) -> Result { + let deadline = self.call_timeout.map(|timeout| Instant::now() + timeout); let request = proto::Request { v: proto::PROTOCOL_VERSION, id, @@ -136,16 +139,57 @@ impl Client { }; let mut line = serde_json::to_vec(&request).map_err(|error| error.to_string())?; line.push(b'\n'); - self.stream - .get_mut() - .write_all(&line) - .map_err(|error| format!("send: {error}"))?; - let mut response_line = String::new(); - self.stream - .read_line(&mut response_line) - .map_err(|error| format!("receive: {error}"))?; + let mut sent = 0; + while sent < line.len() { + self.stream + .get_ref() + .set_write_timeout(remaining(deadline)?) + .map_err(|error| format!("send timeout: {error}"))?; + let count = self + .stream + .get_mut() + .write(line.get(sent..).ok_or("send: invalid offset")?) + .map_err(|error| format!("send: {error}"))?; + if count == 0 { + return Err("send: daemon closed the connection".to_owned()); + } + sent += count; + } + let mut response_line = Vec::new(); + loop { + self.stream + .get_ref() + .set_read_timeout(remaining(deadline)?) + .map_err(|error| format!("receive timeout: {error}"))?; + let available = self + .stream + .fill_buf() + .map_err(|error| format!("receive: {error}"))?; + if available.is_empty() { + return Err("receive: daemon closed the connection".to_owned()); + } + let count = available + .iter() + .position(|byte| *byte == b'\n') + .map_or(available.len(), |index| index + 1); + if response_line.len() + count > 64 * 1024 * 1024 { + return Err("receive: response exceeds 64 MiB".to_owned()); + } + response_line + .extend_from_slice(available.get(..count).ok_or("receive: invalid offset")?); + self.stream.consume(count); + if response_line.last() == Some(&b'\n') { + break; + } + } let response: proto::Response = - serde_json::from_str(&response_line).map_err(|error| format!("decode: {error}"))?; + serde_json::from_slice(&response_line).map_err(|error| format!("decode: {error}"))?; + if response.id != id { + return Err(format!( + "receive: response id {} does not match request {id}", + response.id + )); + } match response.payload { proto::Payload::Ok(reply) => Ok(*reply), proto::Payload::Err { message } => Err(message), @@ -153,6 +197,62 @@ impl Client { } } +fn remaining(deadline: Option) -> Result, String> { + match deadline { + Some(deadline) => { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + Err("daemon call timed out".to_owned()) + } else { + Ok(Some(left)) + } + } + None => Ok(None), + } +} + +#[cfg(all(test, windows))] +mod deadline_tests { + use super::*; + + #[test] + fn call_deadline_bounds_silent_reply() { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let socket = std::path::PathBuf::from(format!("call-{}-{nonce}", std::process::id())); + let name = crate::ipc::endpoint_display(&socket); + let (ready, connected) = std::sync::mpsc::channel(); + let server = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + runtime.block_on(async { + let pipe = tokio::net::windows::named_pipe::ServerOptions::new() + .first_pipe_instance(true) + .create(&name) + .expect("pipe"); + ready.send(()).expect("ready"); + pipe.connect().await.expect("connect"); + tokio::time::sleep(Duration::from_millis(200)).await; + }); + }); + connected.recv().expect("server ready"); + let stream = ClientStream::connect(&socket).expect("client connect"); + let mut client = Client::from_stream(stream).expect("client"); + client.set_deadline(Duration::from_millis(30)); + let started = Instant::now(); + let error = client + .call(proto::Op::Ping) + .expect_err("call should time out"); + assert!(error.contains("timed out"), "{error}"); + assert!(started.elapsed() < Duration::from_millis(180)); + server.join().expect("server exit"); + } +} + fn spawn_daemon(repo_root: &Path, log_path: &Path) -> Result { let exe = std::env::current_exe().map_err(|error| ConnectError::Other(error.to_string()))?; let log = std::fs::File::create(log_path) diff --git a/crates/acyclic/src/ipc.rs b/crates/acyclic/src/ipc.rs index 92a535b..965b1e4 100644 --- a/crates/acyclic/src/ipc.rs +++ b/crates/acyclic/src/ipc.rs @@ -69,6 +69,10 @@ pub struct ClientStream { inner: std::os::unix::net::UnixStream, #[cfg(windows)] inner: std::fs::File, + #[cfg(windows)] + read_timeout: std::cell::Cell>, + #[cfg(windows)] + write_timeout: std::cell::Cell>, } impl ClientStream { @@ -83,6 +87,8 @@ impl ClientStream { } #[cfg(windows)] { + use std::os::windows::fs::OpenOptionsExt as _; + use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OVERLAPPED; // A named pipe is opened like a file. Every server instance being // momentarily busy is normal under concurrent hooks, so a short // bounded retry stands in for WaitNamedPipe. @@ -93,9 +99,16 @@ impl ClientStream { match std::fs::OpenOptions::new() .read(true) .write(true) + .custom_flags(FILE_FLAG_OVERLAPPED) .open(&name) { - Ok(file) => return Ok(Self { inner: file }), + Ok(file) => { + return Ok(Self { + inner: file, + read_timeout: std::cell::Cell::new(None), + write_timeout: std::cell::Cell::new(None), + }); + } Err(error) => { if error.raw_os_error() != Some(BUSY) { return Err(error); @@ -109,9 +122,7 @@ impl ClientStream { } } - /// Bounds a single read. Windows named pipes opened as files carry no - /// per-handle timeout, so this is a no-op there — see the deadline note - /// in `docs/windows-verification.md`. + /// Bounds a single read, including Windows overlapped named-pipe reads. pub fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { #[cfg(unix)] { @@ -119,12 +130,12 @@ impl ClientStream { } #[cfg(windows)] { - let _ = timeout; + self.read_timeout.set(timeout); Ok(()) } } - /// Bounds a single write. No-op on Windows, as for reads. + /// Bounds a single write, including Windows overlapped named-pipe writes. pub fn set_write_timeout(&self, timeout: Option) -> io::Result<()> { #[cfg(unix)] { @@ -132,7 +143,7 @@ impl ClientStream { } #[cfg(windows)] { - let _ = timeout; + self.write_timeout.set(timeout); Ok(()) } } @@ -140,16 +151,188 @@ impl ClientStream { impl io::Read for ClientStream { fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.inner.read(buf) + #[cfg(unix)] + { + self.inner.read(buf) + } + #[cfg(windows)] + { + pipe_io( + &self.inner, + buf.as_mut_ptr(), + buf.len(), + self.read_timeout.get(), + true, + ) + } } } impl io::Write for ClientStream { fn write(&mut self, buf: &[u8]) -> io::Result { - self.inner.write(buf) + #[cfg(unix)] + { + self.inner.write(buf) + } + #[cfg(windows)] + { + pipe_io( + &self.inner, + buf.as_ptr().cast_mut(), + buf.len(), + self.write_timeout.get(), + false, + ) + } } fn flush(&mut self) -> io::Result<()> { - self.inner.flush() + #[cfg(unix)] + { + self.inner.flush() + } + #[cfg(windows)] + { + Ok(()) + } + } +} + +#[cfg(windows)] +#[allow( + unsafe_code, + reason = "owns the event and waits for completion or cancellation before releasing the caller's buffer and OVERLAPPED" +)] +fn pipe_io( + file: &std::fs::File, + buffer: *mut u8, + len: usize, + timeout: Option, + read: bool, +) -> io::Result { + use std::os::windows::io::AsRawHandle as _; + use windows_sys::Win32::Foundation::{CloseHandle, ERROR_IO_PENDING, WAIT_TIMEOUT}; + use windows_sys::Win32::Storage::FileSystem::{ReadFile, WriteFile}; + use windows_sys::Win32::System::Threading::{CreateEventW, INFINITE}; + use windows_sys::Win32::System::IO::{ + CancelIoEx, GetOverlappedResult, GetOverlappedResultEx, OVERLAPPED, + }; + + if len == 0 { + return Ok(0); + } + let handle = file.as_raw_handle().cast(); + let event = unsafe { CreateEventW(std::ptr::null(), 1, 0, std::ptr::null()) }; + if event.is_null() { + return Err(io::Error::last_os_error()); + } + struct Event(windows_sys::Win32::Foundation::HANDLE); + impl Drop for Event { + fn drop(&mut self) { + unsafe { CloseHandle(self.0) }; + } + } + let event = Event(event); + let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; + overlapped.hEvent = event.0; + let count = u32::try_from(len).unwrap_or(u32::MAX); + let submitted = if read { + unsafe { + ReadFile( + handle, + buffer, + count, + std::ptr::null_mut(), + &raw mut overlapped, + ) + } + } else { + unsafe { + WriteFile( + handle, + buffer, + count, + std::ptr::null_mut(), + &raw mut overlapped, + ) + } + }; + if submitted == 0 { + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(ERROR_IO_PENDING.cast_signed()) { + return Err(error); + } + } + let milliseconds = timeout.map_or(INFINITE, |duration| { + u32::try_from(duration.as_millis().max(1)).unwrap_or(INFINITE - 1) + }); + let mut transferred = 0; + let completed = unsafe { + GetOverlappedResultEx( + handle, + &raw mut overlapped, + &raw mut transferred, + milliseconds, + 0, + ) + }; + if completed != 0 { + return Ok(transferred as usize); + } + let error = io::Error::last_os_error(); + // A failed wait may still leave the operation pending. Keep the caller's + // buffer, OVERLAPPED, and event alive until cancellation has completed. + unsafe { + CancelIoEx(handle, &raw mut overlapped); + GetOverlappedResult(handle, &raw mut overlapped, &raw mut transferred, 1); + } + if error.raw_os_error() == Some(WAIT_TIMEOUT.cast_signed()) { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "named-pipe operation timed out", + )); + } + Err(error) +} + +#[cfg(all(test, windows))] +mod deadline_tests { + use super::*; + use std::io::Read as _; + use std::time::{Duration, Instant}; + + #[test] + fn named_pipe_read_deadline_is_enforced() { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let socket = std::path::PathBuf::from(format!("deadline-{}-{nonce}", std::process::id())); + let name = pipe_name(&socket); + let (ready, connected) = std::sync::mpsc::channel(); + let server = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + runtime.block_on(async { + let pipe = create_pipe_instance(&name, true).expect("pipe"); + ready.send(()).expect("ready"); + pipe.connect().await.expect("connect"); + tokio::time::sleep(Duration::from_millis(200)).await; + }); + }); + connected.recv().expect("server ready"); + let mut client = ClientStream::connect(&socket).expect("client connect"); + client + .set_read_timeout(Some(Duration::from_millis(30))) + .expect("deadline"); + let started = Instant::now(); + let error = client + .read(&mut [0_u8; 1]) + .expect_err("read should time out"); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert!(started.elapsed() < Duration::from_millis(180)); + server.join().expect("server exit"); } } diff --git a/docs/windows-verification.md b/docs/windows-verification.md index ff3132d..994a871 100644 --- a/docs/windows-verification.md +++ b/docs/windows-verification.md @@ -189,11 +189,11 @@ correct. crash mid-swap is recovered rather than impossible, which is weaker than the APFS/`renameat2` guarantee. -- **No read/write deadlines.** `ClientStream::set_read_timeout` and - `set_write_timeout` are no-ops on Windows: a pipe opened as a `File` - carries no per-handle timeout. The pre-tool hook's deadline therefore does - not bound anything there. Closing this needs overlapped I/O or a watchdog - thread. **The latency gate's guarantee does not hold on Windows.** +- **Client call deadlines.** Windows named-pipe clients use overlapped I/O + with a completion wait and cancellation. The call timeout covers the whole + request and response, including partial writes and replies. A Windows + regression test holds a pipe read past its deadline and verifies that the + client returns promptly. - **A speculative run's timeout kills without a grace period, and a descendant can escape the job.** `spec_runner` puts each run in a job From 07ff70e3f760732b3fa4bd3bd6bdb7bced6c05c5 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 07:54:49 +0100 Subject: [PATCH 003/106] Use IOCP pipe client and invalidate failed calls --- crates/acyclic/Cargo.toml | 2 - crates/acyclic/src/client.rs | 25 ++++-- crates/acyclic/src/ipc.rs | 158 ++++++++--------------------------- docs/windows-verification.md | 9 +- 4 files changed, 59 insertions(+), 135 deletions(-) diff --git a/crates/acyclic/Cargo.toml b/crates/acyclic/Cargo.toml index 733543c..285a10b 100644 --- a/crates/acyclic/Cargo.toml +++ b/crates/acyclic/Cargo.toml @@ -29,11 +29,9 @@ libc = "0.2" # timeout reach the child's own children (`spec_runner::RunGroup`). windows-sys = { version = "0.61", features = [ "Win32_Foundation", - "Win32_Storage_FileSystem", "Win32_Security", "Win32_Security_Authorization", "Win32_System_JobObjects", - "Win32_System_IO", "Win32_System_Memory", "Win32_System_Threading", ] } diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index 1113b1c..c7166e2 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -26,6 +26,7 @@ pub struct Client { stream: BufReader, next_id: u64, call_timeout: Option, + usable: bool, } #[derive(Debug)] @@ -93,6 +94,7 @@ impl Client { stream: BufReader::new(stream), next_id: 1, call_timeout: None, + usable: true, }) } @@ -103,13 +105,23 @@ impl Client { } pub fn call(&mut self, op: proto::Op) -> Result { + if !self.usable { + return Err("daemon connection requires reconnect after a transport failure".into()); + } let id = self.next_id; self.next_id += 1; let name = format!("{op:?}"); let name = name.split([' ', '{', '(']).next().unwrap_or("?").to_owned(); let started = std::time::Instant::now(); acyclic_engine::trace!("client", "call #{id} {name}"); - let result = self.call_inner(id, op); + let result = match self.call_inner(id, op) { + Ok(proto::Payload::Ok(reply)) => Ok(*reply), + Ok(proto::Payload::Err { message }) => Err(message), + Err(error) => { + self.usable = false; + Err(error) + } + }; match &result { Ok(reply) => { let reply_name = format!("{reply:?}"); @@ -130,7 +142,7 @@ impl Client { result } - fn call_inner(&mut self, id: u64, op: proto::Op) -> Result { + fn call_inner(&mut self, id: u64, op: proto::Op) -> Result { let deadline = self.call_timeout.map(|timeout| Instant::now() + timeout); let request = proto::Request { v: proto::PROTOCOL_VERSION, @@ -190,10 +202,7 @@ impl Client { response.id )); } - match response.payload { - proto::Payload::Ok(reply) => Ok(*reply), - proto::Payload::Err { message } => Err(message), - } + Ok(response.payload) } } @@ -249,6 +258,10 @@ mod deadline_tests { .expect_err("call should time out"); assert!(error.contains("timed out"), "{error}"); assert!(started.elapsed() < Duration::from_millis(180)); + let retry = client + .call(proto::Op::Ping) + .expect_err("connection is poisoned"); + assert!(retry.contains("requires reconnect"), "{retry}"); server.join().expect("server exit"); } } diff --git a/crates/acyclic/src/ipc.rs b/crates/acyclic/src/ipc.rs index 965b1e4..b8f197c 100644 --- a/crates/acyclic/src/ipc.rs +++ b/crates/acyclic/src/ipc.rs @@ -68,7 +68,9 @@ pub struct ClientStream { #[cfg(unix)] inner: std::os::unix::net::UnixStream, #[cfg(windows)] - inner: std::fs::File, + inner: tokio::net::windows::named_pipe::NamedPipeClient, + #[cfg(windows)] + runtime: tokio::runtime::Runtime, #[cfg(windows)] read_timeout: std::cell::Cell>, #[cfg(windows)] @@ -87,24 +89,27 @@ impl ClientStream { } #[cfg(windows)] { - use std::os::windows::fs::OpenOptionsExt as _; - use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OVERLAPPED; + use tokio::net::windows::named_pipe::ClientOptions; // A named pipe is opened like a file. Every server instance being // momentarily busy is normal under concurrent hooks, so a short // bounded retry stands in for WaitNamedPipe. const BUSY: i32 = 231; // ERROR_PIPE_BUSY let name = pipe_name(socket); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .build()?; let mut last = None; for _ in 0..20 { - match std::fs::OpenOptions::new() - .read(true) - .write(true) - .custom_flags(FILE_FLAG_OVERLAPPED) - .open(&name) - { - Ok(file) => { + let opened = { + let _entered = runtime.enter(); + ClientOptions::new().open(&name) + }; + match opened { + Ok(inner) => { return Ok(Self { - inner: file, + inner, + runtime, read_timeout: std::cell::Cell::new(None), write_timeout: std::cell::Cell::new(None), }); @@ -157,13 +162,16 @@ impl io::Read for ClientStream { } #[cfg(windows)] { - pipe_io( - &self.inner, - buf.as_mut_ptr(), - buf.len(), - self.read_timeout.get(), - true, - ) + use tokio::io::AsyncReadExt as _; + match self.read_timeout.get() { + Some(timeout) => self + .runtime + .block_on(async { tokio::time::timeout(timeout, self.inner.read(buf)).await }) + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "named-pipe read timed out") + })?, + None => self.runtime.block_on(async { self.inner.read(buf).await }), + } } } } @@ -176,13 +184,16 @@ impl io::Write for ClientStream { } #[cfg(windows)] { - pipe_io( - &self.inner, - buf.as_ptr().cast_mut(), - buf.len(), - self.write_timeout.get(), - false, - ) + use tokio::io::AsyncWriteExt as _; + match self.write_timeout.get() { + Some(timeout) => self + .runtime + .block_on(async { tokio::time::timeout(timeout, self.inner.write(buf)).await }) + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "named-pipe write timed out") + })?, + None => self.runtime.block_on(async { self.inner.write(buf).await }), + } } } fn flush(&mut self) -> io::Result<()> { @@ -197,103 +208,6 @@ impl io::Write for ClientStream { } } -#[cfg(windows)] -#[allow( - unsafe_code, - reason = "owns the event and waits for completion or cancellation before releasing the caller's buffer and OVERLAPPED" -)] -fn pipe_io( - file: &std::fs::File, - buffer: *mut u8, - len: usize, - timeout: Option, - read: bool, -) -> io::Result { - use std::os::windows::io::AsRawHandle as _; - use windows_sys::Win32::Foundation::{CloseHandle, ERROR_IO_PENDING, WAIT_TIMEOUT}; - use windows_sys::Win32::Storage::FileSystem::{ReadFile, WriteFile}; - use windows_sys::Win32::System::Threading::{CreateEventW, INFINITE}; - use windows_sys::Win32::System::IO::{ - CancelIoEx, GetOverlappedResult, GetOverlappedResultEx, OVERLAPPED, - }; - - if len == 0 { - return Ok(0); - } - let handle = file.as_raw_handle().cast(); - let event = unsafe { CreateEventW(std::ptr::null(), 1, 0, std::ptr::null()) }; - if event.is_null() { - return Err(io::Error::last_os_error()); - } - struct Event(windows_sys::Win32::Foundation::HANDLE); - impl Drop for Event { - fn drop(&mut self) { - unsafe { CloseHandle(self.0) }; - } - } - let event = Event(event); - let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; - overlapped.hEvent = event.0; - let count = u32::try_from(len).unwrap_or(u32::MAX); - let submitted = if read { - unsafe { - ReadFile( - handle, - buffer, - count, - std::ptr::null_mut(), - &raw mut overlapped, - ) - } - } else { - unsafe { - WriteFile( - handle, - buffer, - count, - std::ptr::null_mut(), - &raw mut overlapped, - ) - } - }; - if submitted == 0 { - let error = io::Error::last_os_error(); - if error.raw_os_error() != Some(ERROR_IO_PENDING.cast_signed()) { - return Err(error); - } - } - let milliseconds = timeout.map_or(INFINITE, |duration| { - u32::try_from(duration.as_millis().max(1)).unwrap_or(INFINITE - 1) - }); - let mut transferred = 0; - let completed = unsafe { - GetOverlappedResultEx( - handle, - &raw mut overlapped, - &raw mut transferred, - milliseconds, - 0, - ) - }; - if completed != 0 { - return Ok(transferred as usize); - } - let error = io::Error::last_os_error(); - // A failed wait may still leave the operation pending. Keep the caller's - // buffer, OVERLAPPED, and event alive until cancellation has completed. - unsafe { - CancelIoEx(handle, &raw mut overlapped); - GetOverlappedResult(handle, &raw mut overlapped, &raw mut transferred, 1); - } - if error.raw_os_error() == Some(WAIT_TIMEOUT.cast_signed()) { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "named-pipe operation timed out", - )); - } - Err(error) -} - #[cfg(all(test, windows))] mod deadline_tests { use super::*; diff --git a/docs/windows-verification.md b/docs/windows-verification.md index 994a871..0755be1 100644 --- a/docs/windows-verification.md +++ b/docs/windows-verification.md @@ -189,11 +189,10 @@ correct. crash mid-swap is recovered rather than impossible, which is weaker than the APFS/`renameat2` guarantee. -- **Client call deadlines.** Windows named-pipe clients use overlapped I/O - with a completion wait and cancellation. The call timeout covers the whole - request and response, including partial writes and replies. A Windows - regression test holds a pipe read past its deadline and verifies that the - client returns promptly. +- **Client call deadlines.** Windows named-pipe clients use Tokio's IOCP + transport. The call timeout covers the whole request and response, including + partial writes and replies. A timed-out client must reconnect before another + call so a late daemon response cannot be paired with the wrong request. - **A speculative run's timeout kills without a grace period, and a descendant can escape the job.** `spec_runner` puts each run in a job From e7e661a0462f804732553d28506675a56f276313 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 08:11:28 +0100 Subject: [PATCH 004/106] Fold daemon protocol into the binary crate --- CONTRIBUTING.md | 2 +- Cargo.lock | 9 --------- Cargo.toml | 1 - README.md | 2 +- crates/acyclic-proto/Cargo.toml | 16 ---------------- crates/acyclic/Cargo.toml | 1 - crates/acyclic/src/brief.rs | 2 +- crates/acyclic/src/client.rs | 2 +- crates/acyclic/src/hook.rs | 2 +- crates/acyclic/src/main.rs | 2 +- crates/acyclic/src/mcp.rs | 4 ++-- .../src/lib.rs => acyclic/src/proto.rs} | 0 crates/acyclic/src/server.rs | 2 +- crates/acyclic/src/speculate.rs | 2 +- docs/design/06-installation.md | 2 +- docs/design/implementation-rewind.md | 5 ++--- 16 files changed, 13 insertions(+), 41 deletions(-) delete mode 100644 crates/acyclic-proto/Cargo.toml rename crates/{acyclic-proto/src/lib.rs => acyclic/src/proto.rs} (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c9fbc54..a3d0341 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,7 +100,7 @@ guards what those can't express. The rules, and why each exists: The public product name is defined once, in `product.toml`, and threaded through everywhere else: `product::NAME` in Rust, `scripts/product.sh` in shell. Never hardcode the name as a literal string or path in source — `scripts/check-product-name.sh` fails CI if it drifts. Crate names -(`acyclic`, `acyclic-engine`, `acyclic-proto`, `acyclic-qual`) are internal identifiers and are +(`acyclic`, `acyclic-engine`, `acyclic-qual`) are internal identifiers and are exempt from this check. ## Design context diff --git a/Cargo.lock b/Cargo.lock index d6e40f4..2c9e7bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,6 @@ version = "0.0.2" dependencies = [ "acyclic-engine", "acyclic-fs", - "acyclic-proto", "clap", "libc", "rmcp", @@ -86,14 +85,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "acyclic-proto" -version = "0.0.2" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "acyclic-qual" version = "0.0.2" diff --git a/Cargo.toml b/Cargo.toml index 126b8cf..b5e412b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,6 @@ resolver = "2" members = [ "crates/acyclic", "crates/acyclic-engine", - "crates/acyclic-proto", "crates/acyclic-qual", ] diff --git a/README.md b/README.md index 95f82e1..f735fd8 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ V1 is entirely local: no sandboxes, no managed sessions, no cloud sync. It ships One engine, thin adapters: - **`acyclic` CLI + daemon** — watcher, Merkle-DAG snapshot store, index. Host-agnostic. -- **Per-host adapters** — hook-based for CLIs with a lifecycle-hook API (Claude Code, Codex, Cursor), MCP-based for desktop apps and IDEs without one (Claude Desktop, VS Code; Cursor gets both). Every adapter is a `HostAdapter` in `crates/acyclic/src/install.rs`; the MCP server itself is `crates/acyclic/src/mcp.rs`, a thin translation of each tool call into the same `acyclic-proto::Op` the hooks send. The table under [Install](#per-host) says what each one writes and how far it has been verified; `docs/design/06-installation.md` has the design and the ship decision for the MCP path. +- **Per-host adapters** — hook-based for CLIs with a lifecycle-hook API (Claude Code, Codex, Cursor), MCP-based for desktop apps and IDEs without one (Claude Desktop, VS Code; Cursor gets both). Every adapter is a `HostAdapter` in `crates/acyclic/src/install.rs`; the MCP server itself is `crates/acyclic/src/mcp.rs`, a thin translation of each tool call into the same private `proto::Op` the hooks send. The table under [Install](#per-host) says what each one writes and how far it has been verified; `docs/design/06-installation.md` has the design and the ship decision for the MCP path. ## Launch plan diff --git a/crates/acyclic-proto/Cargo.toml b/crates/acyclic-proto/Cargo.toml deleted file mode 100644 index 6e04816..0000000 --- a/crates/acyclic-proto/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "acyclic-proto" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -publish = false -description = "CLI <-> daemon wire types: newline-delimited JSON over the store's unix socket" - -[dependencies] -serde = { version = "1", features = ["derive"] } -serde_json = "1" - -[lints] -workspace = true diff --git a/crates/acyclic/Cargo.toml b/crates/acyclic/Cargo.toml index 285a10b..8574fb4 100644 --- a/crates/acyclic/Cargo.toml +++ b/crates/acyclic/Cargo.toml @@ -11,7 +11,6 @@ description = "Agent-native state engine: checkpoints, rewind, and blast-radius [dependencies] acyclic-engine = { path = "../acyclic-engine" } acyclic-fs.workspace = true -acyclic-proto = { path = "../acyclic-proto" } clap = { version = "4", features = ["derive", "env"] } rmcp = { version = "3.3.0", features = ["server", "macros", "transport-io", "schemars"] } schemars = "1" diff --git a/crates/acyclic/src/brief.rs b/crates/acyclic/src/brief.rs index 8d56b88..0a61688 100644 --- a/crates/acyclic/src/brief.rs +++ b/crates/acyclic/src/brief.rs @@ -5,8 +5,8 @@ //! the last session ended, what it changed, which branches it abandoned, and //! the verbs that reach the rest. +use crate::proto; use acyclic_engine::product::NAME; -use acyclic_proto as proto; /// Hard cap on the rendered text, including the trailing newline. pub const BUDGET_BYTES: usize = 1000; diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index c7166e2..3f8416b 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -6,8 +6,8 @@ use std::path::Path; use std::time::{Duration, Instant}; use crate::ipc::ClientStream; +use crate::proto; use acyclic_engine::product::NAME; -use acyclic_proto as proto; #[derive(Clone, Copy)] pub enum Spawn { diff --git a/crates/acyclic/src/hook.rs b/crates/acyclic/src/hook.rs index 6f7e3e1..c189843 100644 --- a/crates/acyclic/src/hook.rs +++ b/crates/acyclic/src/hook.rs @@ -10,8 +10,8 @@ use std::io::Read; use std::path::Path; use std::time::Duration; +use crate::proto; use acyclic_engine::product::NAME; -use acyclic_proto as proto; use crate::client::{Client, ConnectError, Spawn}; diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index 2a2c21f..cd17a9f 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -19,6 +19,7 @@ mod hook; mod install; mod ipc; mod mcp; +mod proto; mod server; mod spec_runner; mod speculate; @@ -27,7 +28,6 @@ use std::path::{Path, PathBuf}; use acyclic_engine::product::{self, NAME}; use acyclic_engine::short_hex; -use acyclic_proto as proto; use clap::{Parser, Subcommand}; use client::{Client, ConnectError, Spawn}; diff --git a/crates/acyclic/src/mcp.rs b/crates/acyclic/src/mcp.rs index 837b849..6fc7181 100644 --- a/crates/acyclic/src/mcp.rs +++ b/crates/acyclic/src/mcp.rs @@ -3,7 +3,7 @@ //! that support it alongside hooks (Cursor). Exposes the same verbs already //! surfaced to every other host via `AGENTS_MD_BLOCK`/`SELF_ROLLBACK_SKILL` //! (see `install.rs`) as MCP tools, translating each call directly into the -//! `acyclic-proto::Op` the daemon already understands. The MCP adapters in +//! `proto::Op` the daemon already understands. The MCP adapters in //! `install.rs` register it with each host; `tests/acceptance/mcp-e2e.sh` //! drives it end-to-end. //! @@ -19,8 +19,8 @@ use std::path::{Path, PathBuf}; +use crate::proto; use acyclic_engine::product::{self, NAME}; -use acyclic_proto as proto; use rmcp::{ handler::server::wrapper::Parameters, model::{ErrorCode, Implementation, ServerCapabilities, ServerInfo}, diff --git a/crates/acyclic-proto/src/lib.rs b/crates/acyclic/src/proto.rs similarity index 100% rename from crates/acyclic-proto/src/lib.rs rename to crates/acyclic/src/proto.rs diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index f898ce0..48266f1 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use crate::ipc; +use crate::proto; use acyclic_engine::config::Config; use acyclic_engine::fork::{ self, ForkMode, MountCapability, PromoteOutcome, SessionResolveOutcome, SharedLocalCheckout, @@ -23,7 +24,6 @@ use acyclic_fs::{ mount_native, mount_native_over_existing, CheckoutMountSource, MountFilesystem, NativeMountRequest, NativeMountSession, RoutedMountSource, }; -use acyclic_proto as proto; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::sync::{Mutex, Notify}; diff --git a/crates/acyclic/src/speculate.rs b/crates/acyclic/src/speculate.rs index 59ac60b..9fddca9 100644 --- a/crates/acyclic/src/speculate.rs +++ b/crates/acyclic/src/speculate.rs @@ -22,13 +22,13 @@ use std::path::PathBuf; use std::time::Duration; +use crate::proto; use acyclic_engine::index::Index; use acyclic_engine::pipeline::PipelineHandle; use acyclic_engine::spec::{ RunId, RunOutcome, SpecEvent as LogEvent, SpecEventRow, SpecHit, SpecKey, SpecKind, SpecMetrics, SpecState, SpecStore, SpeculateConfig, }; -use acyclic_proto as proto; use tokio::sync::{mpsc, oneshot}; /// Bound on the claim path's wait for `spec.db`. Short on purpose: the one diff --git a/docs/design/06-installation.md b/docs/design/06-installation.md index 37f9232..1d7a202 100644 --- a/docs/design/06-installation.md +++ b/docs/design/06-installation.md @@ -348,4 +348,4 @@ of them, so they come first. ## Design commitment -This resolves the mechanism question in favor of **CLI-as-core with host adapters** (not MCP-as-core, not per-host deep builds). That choice is what makes OpenCode and future hosts nearly free. MCP wraps the CLI where a host has no other extension point — Claude Desktop's `acyclic mcp` adapter is exactly that: every MCP tool is a thin translation into the same `acyclic-proto::Op` the CLI and hooks already send the daemon, no engine logic lives in the MCP layer itself. See the Claude Desktop row above for why it's marked experimental rather than promoted to a supported host yet. +This resolves the mechanism question in favor of **CLI-as-core with host adapters** (not MCP-as-core, not per-host deep builds). That choice is what makes OpenCode and future hosts nearly free. MCP wraps the CLI where a host has no other extension point — Claude Desktop's `acyclic mcp` adapter is exactly that: every MCP tool is a thin translation into the same private `proto::Op` the CLI and hooks already send the daemon, no engine logic lives in the MCP layer itself. See the Claude Desktop row above for why it's marked experimental rather than promoted to a supported host yet. diff --git a/docs/design/implementation-rewind.md b/docs/design/implementation-rewind.md index ec2ac3f..3cdef18 100644 --- a/docs/design/implementation-rewind.md +++ b/docs/design/implementation-rewind.md @@ -18,8 +18,7 @@ The engine is imported, not built: `acyclic-fs` + `acyclic-fs-mount` via path de ``` crates/ acyclic-engine/ lib: store, pipeline, index, rewind, diff, config - acyclic-proto/ lib: CLI ↔ daemon message types - acyclic/ bin: CLI + hidden `__daemon` subcommand + acyclic/ bin: CLI + hidden `__daemon` subcommand; private protocol module acyclic-qual/ Phase 0 harness → standing bench suite (exists) adapters/claude-code/ hooks, /rewind command, self-rollback skill tests/acceptance/ one script per acceptance criterion @@ -82,7 +81,7 @@ Single-file restore (`rewind --path`): copy the one file out of the target check **Goal:** the bare-CLI product works end to end. -- **Protocol** (`acyclic-proto`): newline-delimited JSON over the unix socket. Ops: `ping, status, checkpoint, timeline, rewind, diff, session_start, session_end, commit, stop`. +- **Protocol** (`acyclic/src/proto.rs`): newline-delimited JSON over the local daemon transport. Ops: `ping, status, checkpoint, timeline, rewind, diff, session_start, session_end, commit, stop`. The one latency-critical detail: `checkpoint{wait:false}` replies on *enqueue* (~ms — the PostToolUse path); `wait:true` replies when the checkpoint lands (the PreToolUse path). - **Daemon** (`acyclic __daemon`): socket listener + pipeline + janitor (trash TTL, idle-commit timer). Pidfile prevents doubles; SIGTERM drains the queue and commits. - **CLI verbs:** `init`, `checkpoint [-m] [--wait] [--durable]`, `timeline`, `rewind [--path]` (prints blast summary, confirms), `diff [--stat]`, `status`, `stop`, `install `. From 44c1cc9ed66d977451db0b918a7859b5671ef1f2 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 08:21:16 +0100 Subject: [PATCH 005/106] Consolidate plugin engine and wire domain types --- CONTRIBUTING.md | 4 +- Cargo.lock | 23 +-- Cargo.toml | 1 - README.md | 2 +- crates/acyclic-engine/Cargo.toml | 35 ---- crates/acyclic-qual/Cargo.toml | 2 +- crates/acyclic-qual/src/main.rs | 2 +- crates/acyclic/Cargo.toml | 12 +- crates/{acyclic-engine => acyclic}/build.rs | 0 crates/acyclic/src/brief.rs | 4 +- crates/acyclic/src/client.rs | 28 +-- .../{acyclic-engine => acyclic}/src/config.rs | 0 .../{acyclic-engine => acyclic}/src/diff.rs | 32 +++- .../src/exclude.rs | 0 .../{acyclic-engine => acyclic}/src/fork.rs | 0 .../{acyclic-engine => acyclic}/src/guard.rs | 0 crates/acyclic/src/hook.rs | 6 +- .../{acyclic-engine => acyclic}/src/index.rs | 10 +- crates/acyclic/src/install.rs | 2 +- crates/acyclic/src/ipc.rs | 4 +- crates/{acyclic-engine => acyclic}/src/lib.rs | 0 crates/acyclic/src/main.rs | 27 ++- crates/acyclic/src/mcp.rs | 2 +- .../{acyclic-engine => acyclic}/src/merge.rs | 0 .../{acyclic-engine => acyclic}/src/names.rs | 0 .../src/pipeline.rs | 0 .../src/product.rs | 0 crates/acyclic/src/proto.rs | 141 ++++++-------- .../{acyclic-engine => acyclic}/src/rewind.rs | 3 +- crates/acyclic/src/server.rs | 174 ++++++++---------- .../{acyclic-engine => acyclic}/src/spec.rs | 0 crates/acyclic/src/spec_runner.rs | 6 +- crates/acyclic/src/speculate.rs | 36 ++-- .../{acyclic-engine => acyclic}/src/store.rs | 0 .../{acyclic-engine => acyclic}/src/trace.rs | 0 .../{acyclic-engine => acyclic}/tests/fork.rs | 16 +- .../tests/merge.rs | 24 +-- .../tests/pipeline.rs | 20 +- product.toml | 2 +- scripts/check-product-name.sh | 4 +- tests/acceptance/crash.sh | 2 +- 41 files changed, 278 insertions(+), 346 deletions(-) delete mode 100644 crates/acyclic-engine/Cargo.toml rename crates/{acyclic-engine => acyclic}/build.rs (100%) rename crates/{acyclic-engine => acyclic}/src/config.rs (100%) rename crates/{acyclic-engine => acyclic}/src/diff.rs (89%) rename crates/{acyclic-engine => acyclic}/src/exclude.rs (100%) rename crates/{acyclic-engine => acyclic}/src/fork.rs (100%) rename crates/{acyclic-engine => acyclic}/src/guard.rs (100%) rename crates/{acyclic-engine => acyclic}/src/index.rs (99%) rename crates/{acyclic-engine => acyclic}/src/lib.rs (100%) rename crates/{acyclic-engine => acyclic}/src/merge.rs (100%) rename crates/{acyclic-engine => acyclic}/src/names.rs (100%) rename crates/{acyclic-engine => acyclic}/src/pipeline.rs (100%) rename crates/{acyclic-engine => acyclic}/src/product.rs (100%) rename crates/{acyclic-engine => acyclic}/src/rewind.rs (99%) rename crates/{acyclic-engine => acyclic}/src/spec.rs (100%) rename crates/{acyclic-engine => acyclic}/src/store.rs (100%) rename crates/{acyclic-engine => acyclic}/src/trace.rs (100%) rename crates/{acyclic-engine => acyclic}/tests/fork.rs (95%) rename crates/{acyclic-engine => acyclic}/tests/merge.rs (97%) rename crates/{acyclic-engine => acyclic}/tests/pipeline.rs (98%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3d0341..9c8d1a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,7 +76,7 @@ guards what those can't express. The rules, and why each exists: - **No lossy `as` casts** between integer widths or signs (`cast_possible_truncation`, `cast_sign_loss`, `cast_possible_wrap`, `cast_precision_loss`, `cast_lossless`). Use `u64::from`, `i64::try_from(x).unwrap_or(i64::MAX)`, or an `allow` that says why the value - is in range. `acyclic_engine::unix_now()` and `short_hex()` exist so the two most common + is in range. `acyclic::unix_now()` and `short_hex()` exist so the two most common cases are written once. - **Closed sets are enums, not strings.** Anything the CLI parses or the wire carries with a fixed vocabulary — checkpoint kinds, hook events, host names, restore actions, diff change @@ -100,7 +100,7 @@ guards what those can't express. The rules, and why each exists: The public product name is defined once, in `product.toml`, and threaded through everywhere else: `product::NAME` in Rust, `scripts/product.sh` in shell. Never hardcode the name as a literal string or path in source — `scripts/check-product-name.sh` fails CI if it drifts. Crate names -(`acyclic`, `acyclic-engine`, `acyclic-qual`) are internal identifiers and are +(`acyclic`, `acyclic-qual`) are internal identifiers and are exempt from this check. ## Design context diff --git a/Cargo.lock b/Cargo.lock index 2c9e7bc..201d5ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,37 +5,24 @@ version = 4 [[package]] name = "acyclic" version = "0.0.2" -dependencies = [ - "acyclic-engine", - "acyclic-fs", - "clap", - "libc", - "rmcp", - "schemars", - "serde", - "serde_json", - "tempfile", - "tokio", - "windows-sys 0.61.2", -] - -[[package]] -name = "acyclic-engine" -version = "0.0.2" dependencies = [ "acyclic-fs", "blake3", "bytes", + "clap", "diffy", "hex", "libc", + "rmcp", "rusqlite", + "schemars", "serde", "serde_json", "tempfile", "thiserror", "tokio", "toml", + "windows-sys 0.61.2", ] [[package]] @@ -89,7 +76,7 @@ dependencies = [ name = "acyclic-qual" version = "0.0.2" dependencies = [ - "acyclic-engine", + "acyclic", "acyclic-fs", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index b5e412b..8acc2e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,6 @@ resolver = "2" members = [ "crates/acyclic", - "crates/acyclic-engine", "crates/acyclic-qual", ] diff --git a/README.md b/README.md index f735fd8..3d79466 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Not yet covered: an `install` writer for Codex's MCP config (TOML), Kimi Code CL ## The public name -`product.toml` at the repo root holds the public name once. The CLI command, `./config.toml`, the state and config directories, hook commands, skill names, message prefixes, the `_TRACE` and `_HOOK` variables, release asset names, and the npm bin all derive from it at build or packaging time (`crates/acyclic-engine/build.rs`, `scripts/product.sh`, the workflows). Crate names stay `acyclic*` because they are internal. `scripts/install.sh` is fetched standalone and mirrors the name, repo, and npm package; `scripts/check-product-name.sh` fails CI if any of them drifts or if any user-facing Rust string spells the name out. Renaming is: change `product.toml`, update the three mirror lines in `install.sh`, rebuild. +`product.toml` at the repo root holds the public name once. The CLI command, `./config.toml`, the state and config directories, hook commands, skill names, message prefixes, the `_TRACE` and `_HOOK` variables, release asset names, and the npm bin all derive from it at build or packaging time (`crates/acyclic/build.rs`, `scripts/product.sh`, the workflows). Crate names stay `acyclic*` because they are internal. `scripts/install.sh` is fetched standalone and mirrors the name, repo, and npm package; `scripts/check-product-name.sh` fails CI if any of them drifts or if any user-facing Rust string spells the name out. Renaming is: change `product.toml`, update the three mirror lines in `install.sh`, rebuild. ## Configuration diff --git a/crates/acyclic-engine/Cargo.toml b/crates/acyclic-engine/Cargo.toml deleted file mode 100644 index 5bc2900..0000000 --- a/crates/acyclic-engine/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -name = "acyclic-engine" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -publish = false -build = "build.rs" -description = "Rewind engine: capture pipeline, checkpoint index, rewind, and diff over acyclic-fs" - -[dependencies] -acyclic-fs.workspace = true -tokio = { version = "1.48", features = ["rt-multi-thread", "macros", "sync", "time"] } -rusqlite = { version = "0.40", features = ["bundled"] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" -toml = "0.8" -thiserror = "2" -hex = "0.4" -blake3 = "1" -bytes = "1" -diffy = "0.4" - -[target.'cfg(unix)'.dependencies] -libc = "0.2" - -[build-dependencies] -toml = "0.8" - -[dev-dependencies] -tempfile = "3" - -[lints] -workspace = true diff --git a/crates/acyclic-qual/Cargo.toml b/crates/acyclic-qual/Cargo.toml index 506492d..64a3a13 100644 --- a/crates/acyclic-qual/Cargo.toml +++ b/crates/acyclic-qual/Cargo.toml @@ -9,7 +9,7 @@ publish = false description = "Phase 0 qualification harness: proves acyclic-fs capture/restore against the Rewind acceptance criteria" [dependencies] -acyclic-engine = { path = "../acyclic-engine" } +acyclic = { path = "../acyclic" } acyclic-fs.workspace = true tokio.workspace = true diff --git a/crates/acyclic-qual/src/main.rs b/crates/acyclic-qual/src/main.rs index 08114a8..33461f7 100644 --- a/crates/acyclic-qual/src/main.rs +++ b/crates/acyclic-qual/src/main.rs @@ -25,7 +25,7 @@ use std::io::Read; use std::path::{Path, PathBuf}; use std::time::Instant; -use acyclic_engine::store::local_options; +use acyclic::store::local_options; use acyclic_fs::model::{ AccessMode, CheckoutMode, ConsistencyMode, FilesystemProfile, GenerationSelector, Lifecycle, MutationMode, VolumeConfig, diff --git a/crates/acyclic/Cargo.toml b/crates/acyclic/Cargo.toml index 8574fb4..f28e1cb 100644 --- a/crates/acyclic/Cargo.toml +++ b/crates/acyclic/Cargo.toml @@ -6,16 +6,23 @@ license.workspace = true repository.workspace = true homepage.workspace = true publish = false +build = "build.rs" description = "Agent-native state engine: checkpoints, rewind, and blast-radius diff for coding-agent sessions" [dependencies] -acyclic-engine = { path = "../acyclic-engine" } acyclic-fs.workspace = true clap = { version = "4", features = ["derive", "env"] } rmcp = { version = "3.3.0", features = ["server", "macros", "transport-io", "schemars"] } schemars = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +rusqlite = { version = "0.40", features = ["bundled"] } +toml = "0.8" +thiserror = "2" +hex = "0.4" +blake3 = "1" +bytes = "1" +diffy = "0.4" tokio = { version = "1.48", features = ["rt-multi-thread", "macros", "net", "io-util", "sync", "time", "signal", "process"] } [target.'cfg(unix)'.dependencies] @@ -38,5 +45,8 @@ windows-sys = { version = "0.61", features = [ [dev-dependencies] tempfile = "3" +[build-dependencies] +toml = "0.8" + [lints] workspace = true diff --git a/crates/acyclic-engine/build.rs b/crates/acyclic/build.rs similarity index 100% rename from crates/acyclic-engine/build.rs rename to crates/acyclic/build.rs diff --git a/crates/acyclic/src/brief.rs b/crates/acyclic/src/brief.rs index 0a61688..abbf324 100644 --- a/crates/acyclic/src/brief.rs +++ b/crates/acyclic/src/brief.rs @@ -6,7 +6,7 @@ //! the verbs that reach the rest. use crate::proto; -use acyclic_engine::product::NAME; +use acyclic::product::NAME; /// Hard cap on the rendered text, including the trailing newline. pub const BUDGET_BYTES: usize = 1000; @@ -180,7 +180,7 @@ fn short(session_id: &str) -> String { } fn age(at: i64) -> String { - let delta = (acyclic_engine::unix_now() - at).max(0); + let delta = (acyclic::unix_now() - at).max(0); if delta < 60 { format!("{delta}s ago") } else if delta < 3600 { diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index 3f8416b..3fbe2f9 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use crate::ipc::ClientStream; use crate::proto; -use acyclic_engine::product::NAME; +use acyclic::product::NAME; #[derive(Clone, Copy)] pub enum Spawn { @@ -47,17 +47,17 @@ impl Client { ) -> Result { let started = std::time::Instant::now(); if let Ok(stream) = ClientStream::connect(socket) { - acyclic_engine::trace!( + acyclic::trace!( "client", "connected to running daemon at {} in {:.1}ms", crate::ipc::endpoint_display(socket), - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); return Self::from_stream(stream); } match spawn { Spawn::Never => { - acyclic_engine::trace!( + acyclic::trace!( "client", "no daemon at {} and spawning is not allowed here", crate::ipc::endpoint_display(socket) @@ -65,7 +65,7 @@ impl Client { Err(ConnectError::NoDaemon) } Spawn::Allowed | Spawn::AllowedFor(_) => { - acyclic_engine::trace!( + acyclic::trace!( "client", "no daemon at {}: spawning one", crate::ipc::endpoint_display(socket) @@ -76,10 +76,10 @@ impl Client { _ => None, }; let client = wait_for_socket(socket, child, log_path, bound); - acyclic_engine::trace!( + acyclic::trace!( "client", "daemon spawn + socket wait took {:.1}ms", - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); client } @@ -113,7 +113,7 @@ impl Client { let name = format!("{op:?}"); let name = name.split([' ', '{', '(']).next().unwrap_or("?").to_owned(); let started = std::time::Instant::now(); - acyclic_engine::trace!("client", "call #{id} {name}"); + acyclic::trace!("client", "call #{id} {name}"); let result = match self.call_inner(id, op) { Ok(proto::Payload::Ok(reply)) => Ok(*reply), Ok(proto::Payload::Err { message }) => Err(message), @@ -126,16 +126,16 @@ impl Client { Ok(reply) => { let reply_name = format!("{reply:?}"); let reply_name = reply_name.split([' ', '{', '(']).next().unwrap_or("?"); - acyclic_engine::trace!( + acyclic::trace!( "client", "call #{id} {name} -> {reply_name} in {:.1}ms", - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); } - Err(message) => acyclic_engine::trace!( + Err(message) => acyclic::trace!( "client", "call #{id} {name} -> error in {:.1}ms: {}", - acyclic_engine::trace::ms(started), + acyclic::trace::ms(started), message.lines().next().unwrap_or("") ), } @@ -421,10 +421,10 @@ fn wait_for_socket( } } if bound.is_some_and(|bound| started.elapsed() > bound) { - acyclic_engine::trace!( + acyclic::trace!( "client", "daemon still starting after {:.1}ms; not waiting", - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); return Err(ConnectError::Starting); } diff --git a/crates/acyclic-engine/src/config.rs b/crates/acyclic/src/config.rs similarity index 100% rename from crates/acyclic-engine/src/config.rs rename to crates/acyclic/src/config.rs diff --git a/crates/acyclic-engine/src/diff.rs b/crates/acyclic/src/diff.rs similarity index 89% rename from crates/acyclic-engine/src/diff.rs rename to crates/acyclic/src/diff.rs index 8d5d978..ab04523 100644 --- a/crates/acyclic-engine/src/diff.rs +++ b/crates/acyclic/src/diff.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; use acyclic_fs::kernel::{FileKind, NamespacePath}; use acyclic_fs::{CancellationToken, GenerationId, ObjectId, WorkCounters}; +use serde::{Deserialize, Serialize}; use crate::store::{LocalCheckout, Store}; use crate::{EngineError, Result}; @@ -25,14 +26,43 @@ pub struct FileChange { pub file_kind: FileKind, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum ChangeKind { Added, Removed, Modified, + #[serde(rename = "metadata")] MetadataOnly, } +impl ChangeKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Added => "added", + Self::Removed => "removed", + Self::Modified => "modified", + Self::MetadataOnly => "metadata", + } + } + + /// The one-letter marker used by the CLI's diff output. + pub fn tag(self) -> &'static str { + match self { + Self::Added => "A", + Self::Removed => "D", + Self::Modified => "M", + Self::MetadataOnly => "m", + } + } +} + +impl std::fmt::Display for ChangeKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.pad(self.as_str()) + } +} + #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) struct RecordSummary { pub(crate) kind: FileKind, diff --git a/crates/acyclic-engine/src/exclude.rs b/crates/acyclic/src/exclude.rs similarity index 100% rename from crates/acyclic-engine/src/exclude.rs rename to crates/acyclic/src/exclude.rs diff --git a/crates/acyclic-engine/src/fork.rs b/crates/acyclic/src/fork.rs similarity index 100% rename from crates/acyclic-engine/src/fork.rs rename to crates/acyclic/src/fork.rs diff --git a/crates/acyclic-engine/src/guard.rs b/crates/acyclic/src/guard.rs similarity index 100% rename from crates/acyclic-engine/src/guard.rs rename to crates/acyclic/src/guard.rs diff --git a/crates/acyclic/src/hook.rs b/crates/acyclic/src/hook.rs index c189843..79c1b44 100644 --- a/crates/acyclic/src/hook.rs +++ b/crates/acyclic/src/hook.rs @@ -11,7 +11,7 @@ use std::path::Path; use std::time::Duration; use crate::proto; -use acyclic_engine::product::NAME; +use acyclic::product::NAME; use crate::client::{Client, ConnectError, Spawn}; @@ -123,7 +123,7 @@ impl HookEvent { pub fn run(repo: &Path, event: &str) -> i32 { let Some(event) = HookEvent::parse(event) else { - acyclic_engine::trace!("hook", "unknown event {event:?}: ignored"); + acyclic::trace!("hook", "unknown event {event:?}: ignored"); return 0; }; // Reading stdin can't hang the agent: hosts close it after writing. @@ -139,7 +139,7 @@ pub fn run(repo: &Path, event: &str) -> i32 { } else { Spawn::Never }; - acyclic_engine::trace!( + acyclic::trace!( "hook", "event {}: daemon spawn {}; pre-tool waits (bounded), post-tool enqueues (ack before capture)", event.as_arg(), diff --git a/crates/acyclic-engine/src/index.rs b/crates/acyclic/src/index.rs similarity index 99% rename from crates/acyclic-engine/src/index.rs rename to crates/acyclic/src/index.rs index 36589b9..d55bbd7 100644 --- a/crates/acyclic-engine/src/index.rs +++ b/crates/acyclic/src/index.rs @@ -9,6 +9,7 @@ use std::path::Path; use acyclic_fs::{Digest, GenerationId}; use rusqlite::{params, Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; use crate::{EngineError, Result}; @@ -74,7 +75,8 @@ impl CheckpointRow { } /// Why a checkpoint exists. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum CheckpointKind { Baseline, Pre, @@ -124,6 +126,12 @@ impl CheckpointKind { } } +impl std::fmt::Display for CheckpointKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.pad(self.as_str()) + } +} + /// Attribution attached to a new checkpoint. #[derive(Clone, Debug, Default)] pub struct Attribution { diff --git a/crates/acyclic/src/install.rs b/crates/acyclic/src/install.rs index db89691..54d8b93 100644 --- a/crates/acyclic/src/install.rs +++ b/crates/acyclic/src/install.rs @@ -15,7 +15,7 @@ //! README's per-host table is the user-facing version of the same list, //! with how far each adapter has been verified. -use acyclic_engine::product::{self, NAME, NPM_PACKAGE}; +use acyclic::product::{self, NAME, NPM_PACKAGE}; use std::path::{Path, PathBuf}; use serde_json::{json, Value}; diff --git a/crates/acyclic/src/ipc.rs b/crates/acyclic/src/ipc.rs index b8f197c..e6f5679 100644 --- a/crates/acyclic/src/ipc.rs +++ b/crates/acyclic/src/ipc.rs @@ -6,7 +6,7 @@ //! the endpoint naming, the "already running" check, and teardown differ. //! //! Callers name the endpoint with the socket path from -//! [`acyclic_engine::store::Paths::socket`] on every platform. On Windows +//! [`acyclic::store::Paths::socket`] on every platform. On Windows //! that path is never created on disk — it only supplies the stable, //! per-store name the pipe is built from. @@ -58,7 +58,7 @@ fn pipe_name(socket: &Path) -> String { } }) .collect(); - format!(r"\\.\pipe\{}-{key}", acyclic_engine::product::NAME) + format!(r"\\.\pipe\{}-{key}", acyclic::product::NAME) } // ---------------------------------------------------------------- client diff --git a/crates/acyclic-engine/src/lib.rs b/crates/acyclic/src/lib.rs similarity index 100% rename from crates/acyclic-engine/src/lib.rs rename to crates/acyclic/src/lib.rs diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index cd17a9f..ec23771 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -26,8 +26,8 @@ mod speculate; use std::path::{Path, PathBuf}; -use acyclic_engine::product::{self, NAME}; -use acyclic_engine::short_hex; +use acyclic::product::{self, NAME}; +use acyclic::short_hex; use clap::{Parser, Subcommand}; use client::{Client, ConnectError, Spawn}; @@ -256,7 +256,7 @@ fn main() { // root that wedges every stat under it. Force-unmount it first (a no-op // unless a bounded probe shows the mount is genuinely wedged), so the // real tree is back before we read anything. - acyclic_engine::fork::reap_dead_shadow(&repo_arg); + acyclic::fork::reap_dead_shadow(&repo_arg); let repo = repo_arg.canonicalize().unwrap_or_else(|error| { eprintln!("{}: bad repo path: {error}", product::NAME); std::process::exit(1); @@ -379,10 +379,10 @@ fn step_aside() { #[cfg(not(windows))] fn step_aside() {} -fn store_paths(repo: &Path) -> Result { - let config = acyclic_engine::config::Config::load(repo).map_err(|error| error.to_string())?; +fn store_paths(repo: &Path) -> Result { + let config = acyclic::config::Config::load(repo).map_err(|error| error.to_string())?; let stores_root = config.store_dir.as_ref().map(PathBuf::from); - acyclic_engine::store::StorePaths::for_repo(repo, stores_root.as_deref()) + acyclic::store::StorePaths::for_repo(repo, stores_root.as_deref()) .map_err(|error| error.to_string()) } @@ -395,7 +395,7 @@ fn connect(repo: &Path, spawn: Spawn) -> Result { /// One line on what forks and Safe Mode can do here, plus setup steps when /// the host lacks a mount provider. Shown by `init` and `install`. fn print_mount_capability() { - let capability = acyclic_engine::fork::mount_capability(); + let capability = acyclic::fork::mount_capability(); if capability.available { println!( "mounts: {} (forks and Safe Mode available)", @@ -406,7 +406,7 @@ fn print_mount_capability() { "mounts: unavailable ({})", capability.reason.as_deref().unwrap_or("unknown reason") ); - println!("{}", acyclic_engine::fork::mount_setup_hint()); + println!("{}", acyclic::fork::mount_setup_hint()); } } @@ -523,7 +523,7 @@ fn print_speculation(spec: &proto::SpecStatus) { } fn policy(repo: &Path) -> i32 { - match acyclic_engine::config::Config::load(repo) { + match acyclic::config::Config::load(repo) { Ok(config) => { let d = config.decompose; println!("fan_out = {}", d.fan_out); @@ -550,17 +550,16 @@ fn policy(repo: &Path) -> i32 { fn init(repo: &Path) -> i32 { let result = (|| -> Result<(), String> { - let config = - acyclic_engine::config::Config::load(repo).map_err(|error| error.to_string())?; + let config = acyclic::config::Config::load(repo).map_err(|error| error.to_string())?; let stores_root = config.store_dir.as_ref().map(PathBuf::from); - let paths = acyclic_engine::store::StorePaths::for_repo(repo, stores_root.as_deref()) + let paths = acyclic::store::StorePaths::for_repo(repo, stores_root.as_deref()) .map_err(|error| error.to_string())?; if paths.meta().exists() { println!("store already exists at {}", paths.root.display()); } else { let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; runtime - .block_on(acyclic_engine::store::Store::init(repo, paths.clone())) + .block_on(acyclic::store::Store::init(repo, paths.clone())) .map_err(|error| error.to_string())?; println!("store created at {}", paths.root.display()); } @@ -1208,7 +1207,7 @@ pub(crate) fn short_session(session_id: &str) -> String { } fn age(created_at: i64) -> String { - let delta = (acyclic_engine::unix_now() - created_at).max(0); + let delta = (acyclic::unix_now() - created_at).max(0); if delta < 60 { format!("{delta}s ago") } else if delta < 3600 { diff --git a/crates/acyclic/src/mcp.rs b/crates/acyclic/src/mcp.rs index 6fc7181..c468914 100644 --- a/crates/acyclic/src/mcp.rs +++ b/crates/acyclic/src/mcp.rs @@ -20,7 +20,7 @@ use std::path::{Path, PathBuf}; use crate::proto; -use acyclic_engine::product::{self, NAME}; +use acyclic::product::{self, NAME}; use rmcp::{ handler::server::wrapper::Parameters, model::{ErrorCode, Implementation, ServerCapabilities, ServerInfo}, diff --git a/crates/acyclic-engine/src/merge.rs b/crates/acyclic/src/merge.rs similarity index 100% rename from crates/acyclic-engine/src/merge.rs rename to crates/acyclic/src/merge.rs diff --git a/crates/acyclic-engine/src/names.rs b/crates/acyclic/src/names.rs similarity index 100% rename from crates/acyclic-engine/src/names.rs rename to crates/acyclic/src/names.rs diff --git a/crates/acyclic-engine/src/pipeline.rs b/crates/acyclic/src/pipeline.rs similarity index 100% rename from crates/acyclic-engine/src/pipeline.rs rename to crates/acyclic/src/pipeline.rs diff --git a/crates/acyclic-engine/src/product.rs b/crates/acyclic/src/product.rs similarity index 100% rename from crates/acyclic-engine/src/product.rs rename to crates/acyclic/src/product.rs diff --git a/crates/acyclic/src/proto.rs b/crates/acyclic/src/proto.rs index 0e68046..4937305 100644 --- a/crates/acyclic/src/proto.rs +++ b/crates/acyclic/src/proto.rs @@ -8,6 +8,60 @@ use std::str::FromStr; use serde::{Deserialize, Serialize}; +pub use acyclic::diff::ChangeKind; +pub use acyclic::index::CheckpointKind; +pub use acyclic::rewind::RestoreAction; + +#[cfg(test)] +mod domain_wire_tests { + use super::{ChangeKind, CheckpointKind, RestoreAction}; + + #[test] + fn domain_enums_preserve_version_one_wire_names() { + let checkpoint_kinds = [ + (CheckpointKind::Baseline, "baseline"), + (CheckpointKind::Pre, "pre"), + (CheckpointKind::Post, "post"), + (CheckpointKind::Manual, "manual"), + (CheckpointKind::PreRewind, "pre_rewind"), + (CheckpointKind::Recovered, "recovered"), + (CheckpointKind::Failed, "failed"), + (CheckpointKind::Noop, "noop"), + (CheckpointKind::Auto, "auto"), + ]; + for (kind, name) in checkpoint_kinds { + let encoded = serde_json::to_string(&kind).unwrap(); + assert_eq!(encoded, format!("\"{name}\"")); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + kind + ); + } + let changes = [ + (ChangeKind::Added, "added"), + (ChangeKind::Removed, "removed"), + (ChangeKind::Modified, "modified"), + (ChangeKind::MetadataOnly, "metadata"), + ]; + for (kind, name) in changes { + let encoded = serde_json::to_string(&kind).unwrap(); + assert_eq!(encoded, format!("\"{name}\"")); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), kind); + } + for (action, name) in [ + (RestoreAction::Restored, "restored"), + (RestoreAction::Removed, "removed"), + ] { + let encoded = serde_json::to_string(&action).unwrap(); + assert_eq!(encoded, format!("\"{name}\"")); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + action + ); + } + } +} + pub const PROTOCOL_VERSION: u32 = 1; /// The kinds a client may ask for. Bookkeeping kinds (`baseline`, @@ -51,93 +105,6 @@ impl fmt::Display for CheckpointRequestKind { } } -/// Why a checkpoint row exists, as the timeline reports it. Mirrors the -/// engine's `index::CheckpointKind`; the daemon converts between them. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CheckpointKind { - Baseline, - Pre, - Post, - Manual, - PreRewind, - Recovered, - Failed, - Noop, - Auto, -} - -impl CheckpointKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Baseline => "baseline", - Self::Pre => "pre", - Self::Post => "post", - Self::Manual => "manual", - Self::PreRewind => "pre_rewind", - Self::Recovered => "recovered", - Self::Failed => "failed", - Self::Noop => "noop", - Self::Auto => "auto", - } - } -} - -impl fmt::Display for CheckpointKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad(self.as_str()) - } -} - -/// What a single-path restore did to the path. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RestoreAction { - /// The path now has the checkpoint's contents. - Restored, - /// The path was absent at the checkpoint, so it was removed. - Removed, -} - -/// How a path differs between two checkpoints. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ChangeKind { - Added, - Removed, - Modified, - /// Mode or other metadata only; contents are identical. - Metadata, -} - -impl ChangeKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Added => "added", - Self::Removed => "removed", - Self::Modified => "modified", - Self::Metadata => "metadata", - } - } - - /// The one-letter tag `diff` listings use: A / D / M, and `m` for - /// metadata-only, so real blast radius stands out from noise. - pub fn tag(self) -> &'static str { - match self { - Self::Added => "A", - Self::Removed => "D", - Self::Modified => "M", - Self::Metadata => "m", - } - } -} - -impl fmt::Display for ChangeKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad(self.as_str()) - } -} - #[derive(Debug, Serialize, Deserialize)] pub struct Request { /// Protocol version; mismatches are rejected. diff --git a/crates/acyclic-engine/src/rewind.rs b/crates/acyclic/src/rewind.rs similarity index 99% rename from crates/acyclic-engine/src/rewind.rs rename to crates/acyclic/src/rewind.rs index c00550b..d974b23 100644 --- a/crates/acyclic-engine/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -31,7 +31,8 @@ pub struct RewindOutcome { } /// What a single-path restore did to the working tree. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum RestoreAction { /// The path now matches the checkpoint's content. Restored, diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 48266f1..e30f728 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -7,18 +7,18 @@ use std::time::{Duration, Instant}; use crate::ipc; use crate::proto; -use acyclic_engine::config::Config; -use acyclic_engine::fork::{ +use acyclic::config::Config; +use acyclic::fork::{ self, ForkMode, MountCapability, PromoteOutcome, SessionResolveOutcome, SharedLocalCheckout, }; -use acyclic_engine::guard::GuardedMountFilesystem; -use acyclic_engine::index::{Attribution, CheckpointKind, CheckpointRow, Index}; -use acyclic_engine::merge::{self, Entry}; -use acyclic_engine::pipeline::{self, PipelineHandle}; -use acyclic_engine::product::NAME; -use acyclic_engine::spec::SpeculateConfig; -use acyclic_engine::store::{Store, StorePaths}; -use acyclic_engine::{rewind, EngineError}; +use acyclic::guard::GuardedMountFilesystem; +use acyclic::index::{Attribution, CheckpointKind, CheckpointRow, Index}; +use acyclic::merge::{self, Entry}; +use acyclic::pipeline::{self, PipelineHandle}; +use acyclic::product::NAME; +use acyclic::spec::SpeculateConfig; +use acyclic::store::{Store, StorePaths}; +use acyclic::{rewind, EngineError}; use acyclic_fs::model::VolumeConfig; use acyclic_fs::{ mount_native, mount_native_over_existing, CheckoutMountSource, MountFilesystem, @@ -36,7 +36,7 @@ struct ForkState { /// The generation promote is judged against. Starts at the fork's cut /// point; a conflicting promote rebases the fork and moves it to the /// head it was rebased onto. - base: acyclic_engine::GenerationId, + base: acyclic::GenerationId, entry: proto::ForkEntry, /// Copy-mode forks only: the materialized directory the user works in. /// Captured back into `shared` at promote, removed at drop/promote. @@ -60,7 +60,7 @@ struct DrySession { fork_id: String, session_id: String, shared: Arc, - base: acyclic_engine::GenerationId, + base: acyclic::GenerationId, mount: NativeMountSession, } @@ -68,8 +68,8 @@ struct DrySession { /// overlay is already committed to the store under `generation`; nothing /// has touched the real tree yet. struct PendingSession { - generation: acyclic_engine::GenerationId, - base: acyclic_engine::GenerationId, + generation: acyclic::GenerationId, + base: acyclic::GenerationId, label: String, } @@ -177,11 +177,11 @@ pub fn run(repo_root: &Path) -> Result<(), String> { let startup = std::time::Instant::now(); let mut phase = std::time::Instant::now(); let mut lap = |name: &str| { - acyclic_engine::trace!( + acyclic::trace!( "daemon", "startup: {name} {:.1}ms (t+{:.1}ms)", - acyclic_engine::trace::ms(phase), - acyclic_engine::trace::ms(startup) + acyclic::trace::ms(phase), + acyclic::trace::ms(startup) ); phase = std::time::Instant::now(); }; @@ -336,22 +336,22 @@ impl Server { async fn dispatch(&self, op: proto::Op) -> proto::Payload { let name = op_name(&op); let started = std::time::Instant::now(); - acyclic_engine::trace!("daemon", "op {name} received"); + acyclic::trace!("daemon", "op {name} received"); let payload = match self.dispatch_inner(op).await { Ok(reply) => { - acyclic_engine::trace!( + acyclic::trace!( "daemon", "op {name} -> {} in {:.1}ms", reply_name(&reply), - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); proto::Payload::Ok(Box::new(reply)) } Err(message) => { - acyclic_engine::trace!( + acyclic::trace!( "daemon", "op {name} -> error in {:.1}ms: {}", - acyclic_engine::trace::ms(started), + acyclic::trace::ms(started), message.lines().next().unwrap_or("") ); err(message) @@ -445,7 +445,7 @@ impl Server { rewind_target: None, }; if wait { - acyclic_engine::trace!( + acyclic::trace!( "daemon", "checkpoint: WAIT path (reply after the capture lands; durable={durable})" ); @@ -460,14 +460,14 @@ impl Server { Ok(proto::Reply::Checkpoint(proto::CheckpointInfo { row_id: outcome.row_id, generation: hex_generation(outcome.generation), - kind: wire_kind(outcome.kind), + kind: outcome.kind, })) } else { // Enqueue-ack: the hook path. Admission into the FIFO // happens BEFORE the ack, so a stop arriving after the // ack queues behind the capture instead of dropping it. // Failures land in the index as `failed`. - acyclic_engine::trace!( + acyclic::trace!( "daemon", "checkpoint: ENQUEUE path (ack on admission, capture runs behind)" ); @@ -566,7 +566,7 @@ impl Server { id: row.id, generation: hex_generation(row.generation), created_at: row.created_at, - kind: wire_kind(row.kind), + kind: row.kind, published: row.published, session_id: row.session_id, host, @@ -633,10 +633,7 @@ impl Server { Ok(proto::Reply::Restore(proto::RestoreInfo { checkpoint: row_id, path: outcome.path.display().to_string(), - action: match outcome.action { - rewind::RestoreAction::Restored => proto::RestoreAction::Restored, - rewind::RestoreAction::Removed => proto::RestoreAction::Removed, - }, + action: outcome.action, recorded_checkpoint, })) } @@ -805,7 +802,7 @@ impl Server { id: id.clone(), path: root.join(&id).display().to_string(), mode: ForkMode::Mount.as_str().to_owned(), - base: acyclic_engine::generation_hex(seed.base), + base: acyclic::generation_hex(seed.base), created_at: unix_now(), session_id: session_id.clone(), conflict_paths: Vec::new(), @@ -863,18 +860,18 @@ impl Server { paths: files.iter().map(|file| PathBuf::from(&file.path)).collect(), }); fork.base = theirs; - fork.entry.base = acyclic_engine::generation_hex(theirs); + fork.entry.base = acyclic::generation_hex(theirs); fork.entry.conflict_paths = files.iter().map(|file| file.path.clone()).collect(); fork.entry.conflict = Some(proto::ConflictInfo { - base: acyclic_engine::generation_hex(conflict_base), - ours: acyclic_engine::generation_hex(ours), - theirs: acyclic_engine::generation_hex(theirs), + base: acyclic::generation_hex(conflict_base), + ours: acyclic::generation_hex(ours), + theirs: acyclic::generation_hex(theirs), }); let fork_path = fork.entry.path.clone(); self.keep_fork(id, fork).await?; return Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic_engine::generation_hex(theirs), + generation: acyclic::generation_hex(theirs), old_tree: None, warning: String::new(), replayed_paths: 0, @@ -917,7 +914,7 @@ impl Server { kept, moved, } => Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic_engine::generation_hex(generation), + generation: acyclic::generation_hex(generation), old_tree: None, warning: if moved { "the mainline had moved; the fork's paths were merged onto it in place" @@ -934,7 +931,7 @@ impl Server { })), Landed::Nothing { generation, kept } => { Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic_engine::generation_hex(generation), + generation: acyclic::generation_hex(generation), old_tree: None, warning: String::new(), replayed_paths: 0, @@ -993,7 +990,7 @@ impl Server { changes .into_iter() .filter(|change| { - change.change != acyclic_engine::diff::ChangeKind::MetadataOnly + change.change != acyclic::diff::ChangeKind::MetadataOnly }) .map(diff_entry) .collect(), @@ -1075,7 +1072,7 @@ impl Server { generation, old_tree, } => Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic_engine::generation_hex(generation), + generation: acyclic::generation_hex(generation), old_tree: old_tree.map(|path| path.display().to_string()), warning: "reload your editor: open files still point at the replaced tree" .into(), @@ -1130,7 +1127,7 @@ impl Server { }; mount_native_over_existing( NativeMountRequest { - mount_id: acyclic_engine::MountId::new(), + mount_id: acyclic::MountId::new(), volume_id, destination, writable: true, @@ -1177,7 +1174,7 @@ impl Server { id: id.clone(), path: dir.display().to_string(), mode: ForkMode::Copy.as_str().to_owned(), - base: acyclic_engine::generation_hex(seed.base), + base: acyclic::generation_hex(seed.base), created_at: unix_now(), session_id: session_id.clone(), conflict_paths: Vec::new(), @@ -1234,10 +1231,10 @@ impl Server { } let publish_started = std::time::Instant::now(); let head = self.handle.publish_head().await.map_err(stringify)?; - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: publish_head {:.1}ms; {} fork, mainline {} since the fork's base", - acyclic_engine::trace::ms(publish_started), + acyclic::trace::ms(publish_started), if fork.copy_dir.is_some() { "copy" } else { @@ -1259,7 +1256,7 @@ impl Server { label, ) .await; - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: outcome {}", match &landed { @@ -1345,7 +1342,7 @@ impl Server { let session = tokio::task::block_in_place(move || { mount_native( NativeMountRequest { - mount_id: acyclic_engine::MountId::new(), + mount_id: acyclic::MountId::new(), volume_id, destination: dest, writable: true, @@ -1419,8 +1416,8 @@ impl Server { &self, id: &str, overlay: Arc, - base: acyclic_engine::GenerationId, - head: acyclic_engine::GenerationId, + base: acyclic::GenerationId, + head: acyclic::GenerationId, copy_dir: Option<&Path>, label: &str, ) -> Result { @@ -1437,17 +1434,17 @@ impl Server { .snapshot_overlay(Arc::clone(&overlay)) .await .map_err(stringify)?; - let snapshot_ms = acyclic_engine::trace::ms(snapshot_started); + let snapshot_ms = acyclic::trace::ms(snapshot_started); let plan_started = std::time::Instant::now(); let mut plan = self .handle .merge_plan(base, head, snapshot, format!("fork {id}")) .await .map_err(stringify)?; - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: snapshot_overlay {snapshot_ms:.1}ms, merge_plan {:.1}ms", - acyclic_engine::trace::ms(plan_started) + acyclic::trace::ms(plan_started) ); // Gitignored paths (bytecode caches, build output, .env) are not // merge payload: a fork's copy never blocks a promote, the @@ -1461,13 +1458,13 @@ impl Server { let ignore_started = std::time::Instant::now(); let ignored = tokio::task::block_in_place(|| merge::ignored_paths(&self.repo_root, &contested)); - let ignore_ms = acyclic_engine::trace::ms(ignore_started); + let ignore_ms = acyclic::trace::ms(ignore_started); let kept: Vec = plan .keep_mainline_for(&ignored) .iter() .map(|path| path.display().to_string()) .collect(); - acyclic_engine::trace!( + acyclic::trace!( "daemon", "merge plan: take_ours={} take_theirs={} merged={} conflicted={} refused={} kept_mainline={}", plan.take_ours.len(), @@ -1488,7 +1485,7 @@ impl Server { "the working tree moved past the fork's base ({}) and {} path(s) cannot be merged:\n{}\n\ One fork must own those paths: re-fork from the current tree and redo that part, \ or rewind to the base. The fork is untouched", - acyclic_engine::generation_hex(base), + acyclic::generation_hex(base), plan.refusals.len(), lines.join("\n") )); @@ -1546,7 +1543,7 @@ impl Server { .await .map_err(stringify)? }; - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: gitignore check {ignore_ms:.1}ms ({} contested), landing source {} in \ {:.1}ms ({} replayed subtree(s), {} merged file(s))", @@ -1556,7 +1553,7 @@ impl Server { } else { "= built merged generation" }, - acyclic_engine::trace::ms(build_started), + acyclic::trace::ms(build_started), plan.take_ours.len(), plan.merged.len() ); @@ -1586,7 +1583,7 @@ impl Server { rebased, format!( "fork {id} rebased onto {} ({} conflict(s))", - acyclic_engine::short_hex(&acyclic_engine::generation_hex(head)), + acyclic::short_hex(&acyclic::generation_hex(head)), plan.conflicted.len() ), ) @@ -1642,7 +1639,7 @@ impl Server { ) .await .map_err(stringify)?; - let pre_ms = acyclic_engine::trace::ms(land_started); + let pre_ms = acyclic::trace::ms(land_started); // One restore for every landing path: one drain, one timeline row // carrying the promote label, however many paths the fork touched. // publish_head captured the tree just now, so no safety row either. @@ -1669,16 +1666,16 @@ impl Server { landing.len() ) })?; - let restore_ms = acyclic_engine::trace::ms(restore_started); + let restore_ms = acyclic::trace::ms(restore_started); let written = u32::try_from(restored.outcomes.len()).unwrap_or(u32::MAX); let landed_generation = restored.generation; // No inline publish: the landed row is a checkpoint like any other // and the idle timer publishes it. Authority publish is O(tree). - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: landing {written} path(s): record rows {pre_ms:.1}ms, \ restore {restore_ms:.1}ms, land total {:.1}ms", - acyclic_engine::trace::ms(land_started) + acyclic::trace::ms(land_started) ); Ok(Landed::Replayed { generation: landed_generation, @@ -1695,8 +1692,8 @@ impl Server { async fn rebase_fork( &self, id: &str, - snapshot: acyclic_engine::GenerationId, - rebased: acyclic_engine::GenerationId, + snapshot: acyclic::GenerationId, + rebased: acyclic::GenerationId, copy_dir: Option<&Path>, ) -> Result<(), String> { let changed: Vec = content_changes( @@ -2121,7 +2118,7 @@ struct PendingBranch { prompt: Option, checkpoints: i64, /// Generations to diff for the branch's size, when the two differ. - diff: Option<(acyclic_engine::GenerationId, acyclic_engine::GenerationId)>, + diff: Option<(acyclic::GenerationId, acyclic::GenerationId)>, } /// Every branch the session rewound away from, with the work it would take @@ -2174,12 +2171,10 @@ fn abandoned_branches( /// Content changes only. A rewind or restore rewrites mtimes on every path /// it materializes, so metadata-only rows are noise for "what changed". -fn content_changes( - changes: Vec, -) -> Vec { +fn content_changes(changes: Vec) -> Vec { changes .into_iter() - .filter(|change| change.change != acyclic_engine::diff::ChangeKind::MetadataOnly) + .filter(|change| change.change != acyclic::diff::ChangeKind::MetadataOnly) .collect() } @@ -2195,20 +2190,20 @@ fn err(message: String) -> proto::Payload { /// and hands an id passed as raw ASCII back to the user as mojibake. Ids are /// hex, so this is a widening on Windows and a copy everywhere else. fn route_name(id: &str) -> Vec { - acyclic_engine::names::str_to_bytes(id) + acyclic::names::str_to_bytes(id) } fn short_id() -> String { // UUIDv7 leads with timestamp bits (identical across nearby calls); // the tail is the random section. - acyclic_engine::MountId::new().into_bytes()[10..] + acyclic::MountId::new().into_bytes()[10..] .iter() .map(|byte| format!("{byte:02x}")) .collect() } fn unix_now() -> i64 { - acyclic_engine::unix_now() + acyclic::unix_now() } /// How a fork ended up in the real tree. @@ -2217,7 +2212,7 @@ enum Landed { /// `merged` of the paths were produced by a three-way content merge; /// `moved` says whether the mainline had moved past the fork's base. Replayed { - generation: acyclic_engine::GenerationId, + generation: acyclic::GenerationId, paths: u32, merged: u32, kept: Vec, @@ -2225,14 +2220,14 @@ enum Landed { }, /// No content changes to land. Nothing { - generation: acyclic_engine::GenerationId, + generation: acyclic::GenerationId, kept: Vec, }, /// Nothing landed: the fork was rebased onto `theirs` and `files` /// carry conflict markers in the fork workspace. Conflicted { - theirs: acyclic_engine::GenerationId, - ours: acyclic_engine::GenerationId, + theirs: acyclic::GenerationId, + ours: acyclic::GenerationId, files: Vec, kept: Vec, }, @@ -2300,33 +2295,14 @@ fn engine_kind(kind: proto::CheckpointRequestKind) -> CheckpointKind { } } -fn wire_kind(kind: CheckpointKind) -> proto::CheckpointKind { - match kind { - CheckpointKind::Baseline => proto::CheckpointKind::Baseline, - CheckpointKind::Pre => proto::CheckpointKind::Pre, - CheckpointKind::Post => proto::CheckpointKind::Post, - CheckpointKind::Manual => proto::CheckpointKind::Manual, - CheckpointKind::PreRewind => proto::CheckpointKind::PreRewind, - CheckpointKind::Recovered => proto::CheckpointKind::Recovered, - CheckpointKind::Failed => proto::CheckpointKind::Failed, - CheckpointKind::Noop => proto::CheckpointKind::Noop, - CheckpointKind::Auto => proto::CheckpointKind::Auto, - } -} - #[allow( clippy::needless_pass_by_value, reason = "used as `.map(diff_entry)` over an owning iterator" )] -fn diff_entry(change: acyclic_engine::diff::FileChange) -> proto::DiffEntry { +fn diff_entry(change: acyclic::diff::FileChange) -> proto::DiffEntry { proto::DiffEntry { path: change.path.display().to_string(), - change: match change.change { - acyclic_engine::diff::ChangeKind::Added => proto::ChangeKind::Added, - acyclic_engine::diff::ChangeKind::Removed => proto::ChangeKind::Removed, - acyclic_engine::diff::ChangeKind::Modified => proto::ChangeKind::Modified, - acyclic_engine::diff::ChangeKind::MetadataOnly => proto::ChangeKind::Metadata, - }, + change: change.change, file_kind: format!("{:?}", change.file_kind).to_lowercase(), ignored: false, } @@ -2336,7 +2312,7 @@ fn timeline_entry(row: CheckpointRow) -> proto::TimelineEntry { proto::TimelineEntry { id: row.id, created_at: row.created_at, - kind: wire_kind(row.kind), + kind: row.kind, published: row.published, session_id: row.session_id, tool_name: row.tool_name, @@ -2346,8 +2322,8 @@ fn timeline_entry(row: CheckpointRow) -> proto::TimelineEntry { } } -fn hex_generation(generation: acyclic_engine::GenerationId) -> String { - acyclic_engine::generation_hex(generation) +fn hex_generation(generation: acyclic::GenerationId) -> String { + acyclic::generation_hex(generation) } fn directory_bytes(root: &Path) -> u64 { diff --git a/crates/acyclic-engine/src/spec.rs b/crates/acyclic/src/spec.rs similarity index 100% rename from crates/acyclic-engine/src/spec.rs rename to crates/acyclic/src/spec.rs diff --git a/crates/acyclic/src/spec_runner.rs b/crates/acyclic/src/spec_runner.rs index 68c3dea..9887e4c 100644 --- a/crates/acyclic/src/spec_runner.rs +++ b/crates/acyclic/src/spec_runner.rs @@ -25,8 +25,8 @@ use std::time::Duration; // Only the Unix sweep names the product; on Windows there is no sweep. #[cfg(unix)] -use acyclic_engine::product::NAME; -use acyclic_engine::spec::RunOutcome; +use acyclic::product::NAME; +use acyclic::spec::RunOutcome; use tokio::io::AsyncWriteExt; use tokio::process::Command; @@ -67,7 +67,7 @@ impl RunSpace { } fn record(&self, pgid: i32, argv0: &str) { - let line = format!("{pgid} {} {argv0}\n", acyclic_engine::unix_now()); + let line = format!("{pgid} {} {argv0}\n", acyclic::unix_now()); let _ = std::fs::write(&self.pid_file, line); } diff --git a/crates/acyclic/src/speculate.rs b/crates/acyclic/src/speculate.rs index 9fddca9..4c54716 100644 --- a/crates/acyclic/src/speculate.rs +++ b/crates/acyclic/src/speculate.rs @@ -23,9 +23,9 @@ use std::path::PathBuf; use std::time::Duration; use crate::proto; -use acyclic_engine::index::Index; -use acyclic_engine::pipeline::PipelineHandle; -use acyclic_engine::spec::{ +use acyclic::index::Index; +use acyclic::pipeline::PipelineHandle; +use acyclic::spec::{ RunId, RunOutcome, SpecEvent as LogEvent, SpecEventRow, SpecHit, SpecKey, SpecKind, SpecMetrics, SpecState, SpecStore, SpeculateConfig, }; @@ -107,7 +107,7 @@ impl SpecHandle { /// request handlers, several of them on the hook path. pub fn notify(&self, event: SpecEvent) { if let Err(error) = self.events.try_send(event) { - acyclic_engine::trace!("spec", "event dropped: {error}"); + acyclic::trace!("spec", "event dropped: {error}"); } } @@ -155,7 +155,7 @@ impl SpecHandle { }; let info = serde_json::from_str(&hit.body).ok(); let _ = store.log(&event(LogEvent::ClaimHit, Some(hit.lead_ms))); - acyclic_engine::trace!("spec", "brief claimed, {}ms ahead", hit.lead_ms); + acyclic::trace!("spec", "brief claimed, {}ms ahead", hit.lead_ms); info } @@ -299,7 +299,7 @@ pub fn spawn( let spec_db = deps.spec_db.clone(); let thread_config = config.clone(); let thread = std::thread::Builder::new() - .name(format!("{}-speculate", acyclic_engine::product::NAME)) + .name(format!("{}-speculate", acyclic::product::NAME)) // The fs futures a diff pulls in are large; match the pipeline's // headroom rather than the 2 MiB default. .stack_size(32 * 1024 * 1024) @@ -314,7 +314,7 @@ pub fn spawn( Err(error) => { eprintln!( "{}: speculation runtime: {error}; speculation off", - acyclic_engine::product::NAME + acyclic::product::NAME ); return; } @@ -371,7 +371,7 @@ async fn run(config: SpeculateConfig, deps: SpecDeps, mut receiver: mpsc::Receiv let Ok(mut store) = SpecStore::open(&deps.spec_db, SCHEDULER_BUSY_TIMEOUT) else { eprintln!( "{}: speculation cache unavailable; speculation off", - acyclic_engine::product::NAME + acyclic::product::NAME ); return; }; @@ -379,7 +379,7 @@ async fn run(config: SpeculateConfig, deps: SpecDeps, mut receiver: mpsc::Receiv // `running` is the one state that is never retryable. if let Ok(swept) = store.sweep_orphans() { if swept > 0 { - acyclic_engine::trace!("spec", "swept {swept} orphaned run(s) from a dead daemon"); + acyclic::trace!("spec", "swept {swept} orphaned run(s) from a dead daemon"); } } let mut scheduler = Scheduler { @@ -444,7 +444,7 @@ impl Scheduler { self.in_flight = Some(run); return; } - acyclic_engine::trace!("spec", "cancelling the summary run: {cause:?}"); + acyclic::trace!("spec", "cancelling the summary run: {cause:?}"); let _ = run.cancel.send(()); let _ = self.store.finish(run.run, &RunOutcome::Cancelled); self.log(LogEvent::Cancel, &run.session_id, run.turn, None, None); @@ -463,7 +463,7 @@ impl Scheduler { self.in_flight = None; let (event, bytes) = match &report.outcome { RunOutcome::Ready { body } => { - acyclic_engine::trace!( + acyclic::trace!( "spec", "summary for turn {} ready in {}ms", report.turn, @@ -607,7 +607,7 @@ impl Scheduler { let (cancel, cancelled) = oneshot::channel(); let done = done.clone(); let owner = session_id.to_owned(); - acyclic_engine::trace!("spec", "pre-firing the summarizer for turn {turn}"); + acyclic::trace!("spec", "pre-firing the summarizer for turn {turn}"); tokio::spawn(async move { let started = std::time::Instant::now(); let outcome = crate::spec_runner::run(spec, &space, cancelled).await; @@ -642,11 +642,7 @@ fn turn_range( index: &Index, session_id: &str, turn: i64, -) -> Option<( - acyclic_engine::GenerationId, - acyclic_engine::GenerationId, - String, -)> { +) -> Option<(acyclic::GenerationId, acyclic::GenerationId, String)> { let row = index.turn(session_id, turn).ok()??; let (first, last) = (row.first_checkpoint?, row.last_checkpoint?); let base = index.latest_target_before(first).ok()??; @@ -667,9 +663,9 @@ fn turn_range( fn render_prompt( config: &SpeculateConfig, prompt: &str, - changes: &[acyclic_engine::diff::FileChange], + changes: &[acyclic::diff::FileChange], ) -> String { - use acyclic_engine::diff::ChangeKind; + use acyclic::diff::ChangeKind; let instruction = if config.prompt_template.is_empty() { SUMMARY_PROMPT.to_owned() } else { @@ -741,7 +737,7 @@ async fn speculate_brief(config: &SpeculateConfig, deps: &SpecDeps, store: &mut let wall_ms = i64::try_from(started.elapsed().as_millis()).unwrap_or(i64::MAX); match &outcome { RunOutcome::Ready { body } => { - acyclic_engine::trace!("spec", "brief precomputed in {wall_ms}ms"); + acyclic::trace!("spec", "brief precomputed in {wall_ms}ms"); log( store, LogEvent::Ready, diff --git a/crates/acyclic-engine/src/store.rs b/crates/acyclic/src/store.rs similarity index 100% rename from crates/acyclic-engine/src/store.rs rename to crates/acyclic/src/store.rs diff --git a/crates/acyclic-engine/src/trace.rs b/crates/acyclic/src/trace.rs similarity index 100% rename from crates/acyclic-engine/src/trace.rs rename to crates/acyclic/src/trace.rs diff --git a/crates/acyclic-engine/tests/fork.rs b/crates/acyclic/tests/fork.rs similarity index 95% rename from crates/acyclic-engine/tests/fork.rs rename to crates/acyclic/tests/fork.rs index 730cfa6..208338e 100644 --- a/crates/acyclic-engine/tests/fork.rs +++ b/crates/acyclic/tests/fork.rs @@ -14,11 +14,11 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; -use acyclic_engine::config::Config; -use acyclic_engine::fork::{PromoteOutcome, SessionResolveOutcome}; -use acyclic_engine::index::{Attribution, CheckpointKind, Index}; -use acyclic_engine::pipeline::{self, PipelineHandle}; -use acyclic_engine::store::{Store, StorePaths}; +use acyclic::config::Config; +use acyclic::fork::{PromoteOutcome, SessionResolveOutcome}; +use acyclic::index::{Attribution, CheckpointKind, Index}; +use acyclic::pipeline::{self, PipelineHandle}; +use acyclic::store::{Store, StorePaths}; use acyclic_fs::kernel::{LogicalName, NamespacePath}; use acyclic_fs::model::VolumeLimits; use acyclic_fs::{CancellationToken, WorkCounters}; @@ -92,8 +92,8 @@ fn namespace(path: &str) -> NamespacePath { .filter(|part| !part.is_empty()) .map(|part| { LogicalName::new( - acyclic_engine::names::encoding(), - acyclic_engine::names::str_to_bytes(part), + acyclic::names::encoding(), + acyclic::names::str_to_bytes(part), limits.maximum_component_bytes, ) .expect("name") @@ -104,7 +104,7 @@ fn namespace(path: &str) -> NamespacePath { /// Writes into a fork's overlay exactly as a mount callback would: through /// the shared checkout. -async fn write_in_fork(seed: &acyclic_engine::fork::ForkSeed, path: &str, bytes: &[u8]) { +async fn write_in_fork(seed: &acyclic::fork::ForkSeed, path: &str, bytes: &[u8]) { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; guard diff --git a/crates/acyclic-engine/tests/merge.rs b/crates/acyclic/tests/merge.rs similarity index 97% rename from crates/acyclic-engine/tests/merge.rs rename to crates/acyclic/tests/merge.rs index 85791b0..0e4a07c 100644 --- a/crates/acyclic-engine/tests/merge.rs +++ b/crates/acyclic/tests/merge.rs @@ -15,13 +15,13 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; -use acyclic_engine::config::Config; -use acyclic_engine::fork::ForkSeed; -use acyclic_engine::index::{Attribution, CheckpointKind, Index}; -use acyclic_engine::merge::{self, ConflictKind, Entry, Reason}; -use acyclic_engine::pipeline::{self, PipelineHandle}; -use acyclic_engine::store::{Store, StorePaths}; -use acyclic_engine::GenerationId; +use acyclic::config::Config; +use acyclic::fork::ForkSeed; +use acyclic::index::{Attribution, CheckpointKind, Index}; +use acyclic::merge::{self, ConflictKind, Entry, Reason}; +use acyclic::pipeline::{self, PipelineHandle}; +use acyclic::store::{Store, StorePaths}; +use acyclic::GenerationId; use acyclic_fs::kernel::{LogicalName, NamespacePath}; use acyclic_fs::model::VolumeLimits; use acyclic_fs::{CancellationToken, WorkCounters}; @@ -124,8 +124,8 @@ fn namespace(path: &str) -> NamespacePath { .split('/') .map(|component| { LogicalName::new( - acyclic_engine::names::encoding(), - acyclic_engine::names::str_to_bytes(component), + acyclic::names::encoding(), + acyclic::names::str_to_bytes(component), limits.maximum_component_bytes, ) .expect("logical name") @@ -295,7 +295,7 @@ fn plan_and_merge_generation_over_real_generations() { .await .expect("diff") .into_iter() - .filter(|c| c.change != acyclic_engine::diff::ChangeKind::MetadataOnly) + .filter(|c| c.change != acyclic::diff::ChangeKind::MetadataOnly) .map(|c| c.path) .collect(); assert_eq!(changed, plan.landing_paths()); @@ -411,7 +411,7 @@ fn conflicts_are_collected_and_r_carries_markers() { .await .expect("diff") .into_iter() - .filter(|c| c.change != acyclic_engine::diff::ChangeKind::MetadataOnly) + .filter(|c| c.change != acyclic::diff::ChangeKind::MetadataOnly) .map(|c| c.path) .collect(); let roots = merge::subtree_roots(&changes); @@ -437,7 +437,7 @@ fn conflicts_are_collected_and_r_carries_markers() { .await .expect("diff") .into_iter() - .filter(|c| c.change != acyclic_engine::diff::ChangeKind::MetadataOnly) + .filter(|c| c.change != acyclic::diff::ChangeKind::MetadataOnly) .map(|c| c.path) .collect(); assert!(remaining.is_empty(), "overlay must equal R: {remaining:?}"); diff --git a/crates/acyclic-engine/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs similarity index 98% rename from crates/acyclic-engine/tests/pipeline.rs rename to crates/acyclic/tests/pipeline.rs index f52982e..4b99a37 100644 --- a/crates/acyclic-engine/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -11,11 +11,11 @@ use std::path::{Path, PathBuf}; use std::time::Duration; -use acyclic_engine::config::Config; -use acyclic_engine::diff::{self, ChangeKind}; -use acyclic_engine::index::{Attribution, CheckpointKind, Index}; -use acyclic_engine::pipeline; -use acyclic_engine::store::{Store, StorePaths}; +use acyclic::config::Config; +use acyclic::diff::{self, ChangeKind}; +use acyclic::index::{Attribution, CheckpointKind, Index}; +use acyclic::pipeline; +use acyclic::store::{Store, StorePaths}; fn read_only_index(path: &Path) -> Index { Index::open(path).expect("open index") @@ -454,10 +454,7 @@ fn single_path_restore_leaves_the_rest_alone() { .restore_path(target(v1.row_id), "src/main.rs".into()) .await .expect("restore file"); - assert_eq!( - outcome.action, - acyclic_engine::rewind::RestoreAction::Restored - ); + assert_eq!(outcome.action, acyclic::rewind::RestoreAction::Restored); assert_eq!( std::fs::read(root.join("src/main.rs")).expect("read"), b"v1\n" @@ -496,10 +493,7 @@ fn single_path_restore_leaves_the_rest_alone() { .restore_path(target(v1.row_id), "new.txt".into()) .await .expect("restore absent"); - assert_eq!( - outcome.action, - acyclic_engine::rewind::RestoreAction::Removed - ); + assert_eq!(outcome.action, acyclic::rewind::RestoreAction::Removed); assert!(!root.join("new.txt").exists()); #[cfg(unix)] diff --git a/product.toml b/product.toml index 2887b87..de956ac 100644 --- a/product.toml +++ b/product.toml @@ -4,7 +4,7 @@ # commands, skill names, error-message prefixes, the `_TRACE` and # `_HOOK` environment variables, release asset names, and the npm bin. # -# Readers: crates/acyclic-engine/build.rs (Rust, via env!), scripts/product.sh +# Readers: crates/acyclic/build.rs (Rust, via env!), scripts/product.sh # (shell), .github/workflows (awk). scripts/check-product-name.sh fails CI if # the one self-contained file (scripts/install.sh) drifts from this. # diff --git a/scripts/check-product-name.sh b/scripts/check-product-name.sh index c0bea4b..81858fb 100755 --- a/scripts/check-product-name.sh +++ b/scripts/check-product-name.sh @@ -3,7 +3,7 @@ # 1. scripts/install.sh is fetched standalone and mirrors `name`, # `github_repo`, and `npm_package`; they must match exactly. # 2. No user-facing Rust source spells the name out. Only crate/module -# identifiers (acyclic_fs, acyclic_engine, acyclic-fs ...) and comments +# identifiers (acyclic_fs, acyclic-qual, ...) and comments # may contain it; strings, paths, and doc templates go through # `product::NAME` / `product::render`. set -euo pipefail @@ -22,7 +22,7 @@ have_npm="$(awk -F'"' '/^NPM_PACKAGE=/{print $2; exit}' "$ROOT/scripts/install.s # Literal uses of the current name in strings/paths of user-facing crates. # Comments (// and //!) are allowed; identifiers with '_' are crate paths. stray="$(grep -rn --include='*.rs' -E "\"[^\"]*\b${want_name}\b[^\"]*\"|\`${want_name} |\.${want_name}/" \ - "$ROOT/crates/acyclic/src" "$ROOT/crates/acyclic-engine/src" \ + "$ROOT/crates/acyclic/src" \ | grep -v -E "${want_name}[_-](fs|engine|proto|qual)|^[^:]+:[0-9]+:\s*//" || true)" if [ -n "$stray" ]; then echo "hardcoded product name in Rust source (use product::NAME / product::render):" >&2 diff --git a/tests/acceptance/crash.sh b/tests/acceptance/crash.sh index 140c78f..fb129ff 100755 --- a/tests/acceptance/crash.sh +++ b/tests/acceptance/crash.sh @@ -27,7 +27,7 @@ acy checkpoint --wait --kind post >/dev/null || fail "checkpoint after kill -9" # --- crash mid-swap: journal-driven recovery ------------------------------ # The atomic-exchange platforms (macOS/Linux) always leave the repo whole; # a crash there leaves the journal plus a stray tmp tree. The repo-missing -# two-step variant is unit-tested in acyclic-engine (recover()). +# two-step variant is unit-tested in acyclic (recover()). acy stop >/dev/null sleep 0.5 JOURNAL="$(ls -d "$STORES"/*/)"rewind-journal.json From 7927d919559441adee7731125aea836c4078de7c Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 09:11:48 +0100 Subject: [PATCH 006/106] Make daemon readiness wait for pipeline baseline --- crates/acyclic/src/client.rs | 9 ++++----- crates/acyclic/src/pipeline.rs | 16 +++++----------- crates/acyclic/src/server.rs | 8 +++++++- crates/acyclic/tests/pipeline.rs | 21 +++++++++++++++++++++ 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index 3fbe2f9..602050d 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -409,11 +409,10 @@ fn wait_for_socket( loop { if let Ok(stream) = ClientStream::connect(socket) { if let Ok(mut client) = Client::from_stream(stream) { - // The daemon binds its socket before it opens the store, so - // a connect can succeed while the ping waits on the store - // open; bound the ping too so a caller with a bound never - // sits on it. - client.set_deadline(bound.unwrap_or(Duration::from_secs(60))); + // The socket can accept connections before the pipeline has + // completed its baseline. Ping waits for that pipeline and + // carries a startup error back to this caller. + client.set_deadline(deadline.saturating_sub(started.elapsed())); if client.call(proto::Op::Ping).is_ok() { client.set_deadline(Duration::from_secs(24 * 60 * 60)); return Ok(client); diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index b2e6903..93fd63a 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -232,7 +232,7 @@ enum Request { reply: oneshot::Sender>, }, Status { - reply: oneshot::Sender, + reply: oneshot::Sender>, }, SetShadowed { active: bool, @@ -625,7 +625,7 @@ impl PipelineHandle { } pub async fn status(&self) -> Result { - request!(self, Status {}) + request!(self, Status {})? } /// Safe Mode: while a session's fork is shadow-mounted over the real repo @@ -874,13 +874,7 @@ fn fail_request(request: Request, message: &str) { Request::RestorePaths { reply, .. } => drop(reply.send(Err(error()))), Request::TurnStarted { reply, .. } => drop(reply.send(Err(error()))), Request::SetShadowed { reply, .. } => drop(reply.send(Err(error()))), - Request::Status { reply } => drop(reply.send(StatusReport { - state: State::Baselining, - last_checkpoint: None, - unpublished: 0, - checkpoints_since_commit: 0, - watcher: WatcherHealth::default(), - })), + Request::Status { reply } => drop(reply.send(Err(error()))), Request::SessionStarted { reply, .. } | Request::SessionEnded { reply, .. } => { drop(reply.send(Err(error()))); } @@ -1270,13 +1264,13 @@ impl Pipeline { false } Request::Status { reply } => { - let _ = reply.send(StatusReport { + let _ = reply.send(Ok(StatusReport { state: self.state, last_checkpoint: self.last_checkpoint_row, unpublished: self.index.unpublished_count().unwrap_or(0), checkpoints_since_commit: self.checkpoints_since_commit, watcher: self.watcher_health.clone(), - }); + })); false } Request::SetShadowed { active, reply } => { diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index e30f728..0cca91a 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -404,7 +404,13 @@ impl Server { async fn dispatch_inner(&self, op: proto::Op) -> Result { *self.last_activity.lock().await = Instant::now(); match op { - proto::Op::Ping => Ok(proto::Reply::Pong), + proto::Op::Ping => { + let status = self.handle.status().await.map_err(stringify)?; + if status.state == pipeline::State::Baselining { + return Err("pipeline baseline is still in progress".to_owned()); + } + Ok(proto::Reply::Pong) + } proto::Op::Status => { let status = self.handle.status().await.map_err(stringify)?; Ok(proto::Reply::Status(proto::StatusInfo { diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index 4b99a37..e59bcc5 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -33,6 +33,27 @@ fn fast_config() -> Config { } } +#[test] +fn startup_failure_is_reported_by_status() { + let repo = tempfile::tempdir().expect("repo"); + let stores = tempfile::tempdir().expect("stores"); + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("paths"); + let runtime = tokio::runtime::Runtime::new().expect("runtime"); + let store = runtime + .block_on(Store::init(repo.path(), paths.clone())) + .expect("init store"); + let index = Index::open(&paths.index_db()).expect("index"); + std::fs::remove_dir(repo.path()).expect("remove empty repo before watcher opens"); + let (handle, thread) = pipeline::spawn(store, index, fast_config()); + + let error = runtime + .block_on(handle.status()) + .expect_err("startup must fail"); + assert!(error.to_string().contains("pipeline failed to start")); + drop(handle); + thread.join().expect("pipeline thread"); +} + #[test] fn checkpoint_rewind_journey() { let repo = tempfile::tempdir().expect("repo"); From 8e4ee8c0d2faed994964ce1bd810116526a56158 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 09:23:45 +0100 Subject: [PATCH 007/106] Expose baseline phase timing in latency probe --- tests/acceptance/latency.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/acceptance/latency.sh b/tests/acceptance/latency.sh index 5ec83fc..944f883 100755 --- a/tests/acceptance/latency.sh +++ b/tests/acceptance/latency.sh @@ -18,6 +18,9 @@ BUDGET_MS="${ACYCLIC_LAT_BUDGET_MS:-100}" setup_repo "$QUAL" corpus "$R" "$FILES" "$CORPUS_MB" >/dev/null || fail "corpus generation" acy init >/dev/null || fail "init (includes baseline of the corpus)" +if [ "${ACYCLIC_TRACE:-}" = 1 ]; then + grep 'baseline phases:' "$STORES"/*/daemon.log | tail -1 || true +fi # Warm-up round, then timed samples. Each round touches a file so the # checkpoint has real work queued behind the ack. From b86d72899eb39d27955abd312a7a31bdf2473933 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 09:26:20 +0100 Subject: [PATCH 008/106] Return permanent daemon baseline errors immediately --- crates/acyclic/src/client.rs | 14 +++++++++++--- crates/acyclic/src/server.rs | 5 +---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index 602050d..e386748 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -413,9 +413,17 @@ fn wait_for_socket( // completed its baseline. Ping waits for that pipeline and // carries a startup error back to this caller. client.set_deadline(deadline.saturating_sub(started.elapsed())); - if client.call(proto::Op::Ping).is_ok() { - client.set_deadline(Duration::from_secs(24 * 60 * 60)); - return Ok(client); + match client.call(proto::Op::Ping) { + Ok(_) => { + client.set_deadline(Duration::from_secs(24 * 60 * 60)); + return Ok(client); + } + Err(message) if client.usable => { + // A protocol error is the pipeline's completed startup + // result; reconnecting cannot repair that baseline. + return Err(ConnectError::Other(message)); + } + Err(_) => {} } } } diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 0cca91a..809d949 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -405,10 +405,7 @@ impl Server { *self.last_activity.lock().await = Instant::now(); match op { proto::Op::Ping => { - let status = self.handle.status().await.map_err(stringify)?; - if status.state == pipeline::State::Baselining { - return Err("pipeline baseline is still in progress".to_owned()); - } + self.handle.status().await.map_err(stringify)?; Ok(proto::Reply::Pong) } proto::Op::Status => { From 2ba1c94d7f74eb386d5e7b01573b1c941d0325c9 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 10:32:33 +0100 Subject: [PATCH 009/106] Collapse plugin qualification into shared SDK gate --- .github/workflows/ci.yml | 268 +------- Cargo.lock | 9 - Cargo.toml | 1 - crates/acyclic-qual/Cargo.toml | 17 - crates/acyclic-qual/src/main.rs | 1104 ------------------------------- scripts/check-product-name.sh | 2 +- scripts/ci-local.sh | 80 +-- scripts/docker-linux.sh | 41 +- tests/acceptance/common.sh | 3 +- tests/acceptance/run-all.sh | 2 +- 10 files changed, 42 insertions(+), 1485 deletions(-) delete mode 100644 crates/acyclic-qual/Cargo.toml delete mode 100644 crates/acyclic-qual/src/main.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a014c2..66ea622 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,270 +1,56 @@ -name: ci +name: qualification on: push: branches: [main] pull_request: -# A new push to the same PR (or to main) supersedes the run in flight; -# stop paying for the old one. release.yml deliberately does not cancel. concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: qualify-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -env: - FUSE_T_VERSION: 1.2.7 - CARGO_TERM_COLOR: always - # acyclic-fs is fetched from its own git repo (see Cargo.toml); cargo's - # built-in libgit2 fetcher can't resolve a pinned commit SHA that isn't a - # branch tip on that host, so delegate to the system git CLI instead. - CARGO_NET_GIT_FETCH_WITH_CLI: true +permissions: + contents: read jobs: - # Three cheap jobs (changes, deny, lint) run in parallel; the expensive - # test matrix (FUSE-T install, release build, acceptance suite on two - # OSes) only starts once deny and lint pass, so a formatting slip or a - # banned crate is reported in about a minute instead of after a full - # matrix run — and it is skipped outright when nothing that could change - # a test result was touched. - - # Which parts of the tree a change touches. A job-level filter rather - # than a workflow-level `paths-ignore` because branch protection on main - # requires the `test (...)` checks: a workflow that never runs leaves - # them "expected" forever and blocks the merge, whereas a job skipped by - # `if:` reports as skipped, which counts as passing. - changes: - runs-on: ubuntu-24.04 - # paths-filter lists a PR's files through the API; the default token - # here is read-only on contents and nothing else. - permissions: - contents: read - pull-requests: read - outputs: - code: ${{ steps.filter.outputs.code }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - # On a push to main the filter diffs against the pre-push commit, - # which a depth-1 checkout does not have. - fetch-depth: 0 - - id: filter - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 - with: - # "some file changed that is not docs/prose": a negation-only list - # matches nothing, so the positive `**` is required, and - # some-with-excludes makes the negations apply to it. - predicate-quantifier: some-with-excludes - filters: | - code: - - '**' - - '!**/*.md' - - '!docs/**' - - '!LICENSE' - - '!.github/CODEOWNERS' - - # Licenses, advisories, and sources per deny.toml. Keeps the published - # SBOM inside the permissive allowlist and fails on known-vulnerable or - # yanked crates. Always runs: the secrets scan applies to docs too. - deny: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: Product name is single-sourced (product.toml) - run: bash scripts/check-product-name.sh - - name: No forbidden files or credential patterns - run: bash scripts/check-no-secrets.sh - - name: Code quality (line width, TODO format, comment blocks, duplication) - run: bash scripts/check-code-quality.sh - - uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2.1.1 - with: - command: check - arguments: --locked - log-level: warn - - # Formatting and lint. Fast and toolchain-only (no FUSE-T needed). - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: Install toolchain - run: | - rustup toolchain install stable --profile minimal --component rustfmt --component clippy - rustup default stable - - name: Cache cargo - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: lint-cargo-${{ hashFiles('Cargo.lock') }} - - name: cargo fmt --check - run: cargo fmt --all --check - - name: cargo clippy - run: cargo clippy --workspace --all-targets --all-features -- -D warnings - - # Line coverage of the unit and integration tests, as a job summary and - # an lcov artifact, with a floor so a change can't quietly delete tests. - # The daemon, client, and MCP server are exercised by the acceptance - # scripts rather than `cargo test`, so they read as 0% here; raise the - # floor as unit coverage of those grows, not by counting the scripts. - coverage: - needs: [changes, deny, lint] - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: Install toolchain - run: | - rustup toolchain install stable --profile minimal --component llvm-tools-preview - rustup default stable - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 - with: - tool: cargo-llvm-cov - - name: Cache cargo - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: coverage-cargo-${{ hashFiles('Cargo.lock') }} - - name: cargo llvm-cov - run: | - cargo llvm-cov --workspace --all-features --lcov --output-path lcov.info --fail-under-lines 48 - { - echo '## Test coverage (lines)' - echo '```' - cargo llvm-cov report --summary-only - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: lcov - path: lcov.info - - # Windows, which nothing else in this file covers. The lint job runs on - # Linux, so every `#[cfg(windows)]` block in the tree is invisible to it — - # a Windows-only arm can stop compiling and no other check notices. This - # job builds and lints those arms, runs the workspace tests, and drives - # the parts of the product whose behaviour genuinely differs there - # (named-pipe transport, UTF-16LE names, renaming the repo root, copy - # forks). The POSIX acceptance suite is not run: it assumes mount tooling, - # symlinks and modes that Windows does not have. - windows: - needs: [changes, deny, lint] - if: >- - always() - && needs.deny.result == 'success' - && needs.lint.result == 'success' - && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') - runs-on: windows-2022 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: Install toolchain - run: | - rustup toolchain install stable --profile minimal --component rustfmt --component clippy - rustup default stable - - - name: Cache cargo - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: windows-cargo-${{ hashFiles('Cargo.lock') }} - - - name: cargo fmt --check - run: cargo fmt --all --check - - # The point of the job: lint the cfg(windows) arms the Linux lint job - # never compiles. - - name: cargo clippy - run: cargo clippy --workspace --all-targets --all-features -- -D warnings - - - name: Unit and integration tests - run: cargo test --workspace - - - name: Release build - run: cargo build --release --locked -p acyclic - - - name: Windows end-to-end smoke - shell: bash - env: - ACYCLIC_BIN: ${{ github.workspace }}/target/release/acyclic.exe - run: bash tests/acceptance/windows-smoke.sh - - test: - needs: [changes, deny, lint] - # Fail closed: run when the change filter says code moved, and also - # when the filter job itself failed (an empty output must not read as - # "nothing to test", since a skipped required check counts as passing). - # deny and lint failing still skip this job; those are required checks - # in their own right, so the PR stays red. - if: >- - always() - && needs.deny.result == 'success' - && needs.lint.result == 'success' - && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') + qualify: strategy: fail-fast: false matrix: - os: [macos-14, ubuntu-24.04] + os: [ubuntu-24.04, windows-2022, macos-14] runs-on: ${{ matrix.os }} + timeout-minutes: 20 steps: - name: Checkout plugin uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - path: graphcoder-plugin - - - name: Install FUSE-T (macOS) + path: worktrees/graphcoder/plugin-sdk-lab + persist-credentials: false + - name: Checkout SDK + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: acyclic-labs/sdk + ref: codex/consumer-driven-lab + path: sdk + persist-credentials: false + - name: Install FUSE-T on macOS if: runner.os == 'macOS' + shell: bash run: | - set -euo pipefail curl --fail --location --retry 5 \ - "https://github.com/macos-fuse-t/fuse-t/releases/download/${FUSE_T_VERSION}/fuse-t-macos-installer-${FUSE_T_VERSION}.pkg" \ + https://github.com/macos-fuse-t/fuse-t/releases/download/1.2.7/fuse-t-macos-installer-1.2.7.pkg \ --output /tmp/fuse-t.pkg sudo installer -pkg /tmp/fuse-t.pkg -target / - test -d /usr/local/include/fuse3 - - - name: Install toolchain - run: rustup toolchain install stable --profile minimal && rustup default stable - - - name: Cache cargo + - name: Cache immutable dependencies and build outputs uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cargo/registry ~/.cargo/git - graphcoder-plugin/target - key: ${{ matrix.os }}-cargo-${{ hashFiles('graphcoder-plugin/Cargo.lock') }} - - - name: Unit and integration tests - working-directory: graphcoder-plugin - run: cargo test --workspace - - - name: Release build (acceptance + latency gate run against it) - working-directory: graphcoder-plugin - run: cargo build --release - - - name: Acceptance suite - working-directory: graphcoder-plugin - env: - ACYCLIC_BIN: ${{ github.workspace }}/graphcoder-plugin/target/release/acyclic - ACYCLIC_QUAL: ${{ github.workspace }}/graphcoder-plugin/target/release/acyclic-qual - # Smaller corpus keeps CI wall-clock sane; the budget is unchanged. - ACYCLIC_LAT_FILES: "5000" - ACYCLIC_LAT_MB: "64" - ACYCLIC_SOAK_ROUNDS: "30" - run: bash tests/acceptance/run-all.sh + sdk/target-* + key: qualify-${{ matrix.os }}-${{ hashFiles('sdk/Cargo.lock', 'worktrees/graphcoder/plugin-sdk-lab/Cargo.lock') }} + restore-keys: | + qualify-${{ matrix.os }}- + - name: Run the shared bounded gate + shell: pwsh + run: python sdk/scripts/qualify-local.py --plugin-root worktrees/graphcoder/plugin-sdk-lab\n \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 201d5ac..9ff3a3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,15 +72,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "acyclic-qual" -version = "0.0.2" -dependencies = [ - "acyclic", - "acyclic-fs", - "tokio", -] - [[package]] name = "acyclic-stream" version = "1.0.0-rc.7" diff --git a/Cargo.toml b/Cargo.toml index 8acc2e5..3c796f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,6 @@ resolver = "2" members = [ "crates/acyclic", - "crates/acyclic-qual", ] [workspace.package] diff --git a/crates/acyclic-qual/Cargo.toml b/crates/acyclic-qual/Cargo.toml deleted file mode 100644 index 64a3a13..0000000 --- a/crates/acyclic-qual/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "acyclic-qual" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -publish = false -description = "Phase 0 qualification harness: proves acyclic-fs capture/restore against the Rewind acceptance criteria" - -[dependencies] -acyclic = { path = "../acyclic" } -acyclic-fs.workspace = true -tokio.workspace = true - -[lints] -workspace = true diff --git a/crates/acyclic-qual/src/main.rs b/crates/acyclic-qual/src/main.rs deleted file mode 100644 index 33461f7..0000000 --- a/crates/acyclic-qual/src/main.rs +++ /dev/null @@ -1,1104 +0,0 @@ -//! Phase 0 qualification harness for the Rewind milestone. -//! -//! Subcommands: -//! fixture [--with-fifo] build a small fixture repo exercising the tricky cases -//! roundtrip capture into a fresh store under , commit, -//! materialize into /restore, compare content+mode -//! -//! Exit code 0 = round-trip verified identical; 1 = mismatches or engine failure. - -#![allow( - clippy::indexing_slicing, - clippy::string_slice, - clippy::panic, - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::cast_precision_loss, - reason = "qualification harness, not shipped: a bad invocation or a broken \ - invariant should crash with its message rather than be handled" -)] -#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] - -use std::collections::BTreeMap; -use std::fs; -use std::io::Read; -use std::path::{Path, PathBuf}; -use std::time::Instant; - -use acyclic::store::local_options; -use acyclic_fs::model::{ - AccessMode, CheckoutMode, ConsistencyMode, FilesystemProfile, GenerationSelector, Lifecycle, - MutationMode, VolumeConfig, -}; -use acyclic_fs::{ - capture_baseline, capture_root_identity, materialize_checkout, CaptureOptions, - MaterializeOptions, -}; -use acyclic_fs::{ - CancellationToken, CheckoutCommitOutcome, GenerationId, LocalFs, OperationId, VolumeId, - WorkCounters, -}; - -fn main() { - let args: Vec = std::env::args().skip(1).collect(); - let result = match args.first().map(String::as_str) { - Some("fixture") => fixture(&args[1..]), - Some("roundtrip") => roundtrip(&args[1..]), - Some("corpus") => corpus(&args[1..]), - Some("bench") => bench(&args[1..]), - Some("restore-gen") => restore_gen(&args[1..]), - Some("mount-smoke") => mount_smoke(&args[1..]), - Some("mount-smoke2") => mount_smoke2(&args[1..]), - Some("mount-hold") => mount_hold(&args[1..]), - Some("source-probe") => source_probe(&args[1..]), - _ => Err( - "usage: acyclic-qual fixture [--with-fifo] | roundtrip \ - | corpus | bench [rounds]" - .into(), - ), - }; - if let Err(message) = result { - eprintln!("FAIL: {message}"); - std::process::exit(1); - } -} - -type Failure = Box; - -fn engine_err(context: &str) -> impl FnOnce(E) -> Failure + '_ { - move |error| format!("{context}: {error:?}").into() -} - -// --------------------------------------------------------------------------- -// fixture -// --------------------------------------------------------------------------- - -fn fixture(args: &[String]) -> Result<(), Failure> { - let root = PathBuf::from(args.first().ok_or("fixture: missing ")?); - let with_fifo = args.iter().any(|a| a == "--with-fifo"); - fs::create_dir_all(root.join("src/nested"))?; - fs::create_dir_all(root.join("node_modules/.bin"))?; - fs::create_dir_all(root.join(".git/objects"))?; - fs::create_dir_all(root.join("empty-dir"))?; - - fs::write(root.join("README.md"), b"fixture repo\n")?; - fs::write(root.join(".gitignore"), b"generated/\n.env\n")?; - fs::write(root.join(".env"), b"SECRET=hunter2\n")?; - fs::write(root.join("src/main.rs"), b"fn main() {}\n")?; - fs::write(root.join("src/nested/mod.rs"), b"// nested\n")?; - fs::write(root.join(".git/objects/pack-data"), b"\x00\x01\x02git\n")?; - fs::write(root.join("uni-\u{00e9}\u{4e2d}.txt"), b"unicode name\n")?; - - // Deterministic 8 MiB binary file (multi-chunk content). - let mut big = Vec::with_capacity(8 * 1024 * 1024); - let mut state: u32 = 0x9e37_79b9; - while big.len() < 8 * 1024 * 1024 { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - big.extend_from_slice(&state.to_le_bytes()); - } - fs::write(root.join("assets.bin"), &big)?; - - // Executable script (mode-bit round-trip). - let script = root.join("tool.sh"); - fs::write(&script, b"#!/bin/sh\necho ok\n")?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&script, fs::Permissions::from_mode(0o755))?; - } - - // Symlinks: valid relative (the node_modules/.bin shape) and dangling. - #[cfg(unix)] - { - use std::os::unix::fs::symlink; - symlink("../../tool.sh", root.join("node_modules/.bin/tool"))?; - symlink("no-such-target", root.join("dangling-link"))?; - } - - // Hard-link pair. - fs::write(root.join("hardlink-a"), b"same inode\n")?; - fs::hard_link(root.join("hardlink-a"), root.join("hardlink-b"))?; - - if with_fifo { - let status = std::process::Command::new("mkfifo") - .arg(root.join("pipe.fifo")) - .status()?; - if !status.success() { - return Err("mkfifo failed".into()); - } - } - - println!("fixture written to {}", root.display()); - Ok(()) -} - -// --------------------------------------------------------------------------- -// restore-gen: materialize one generation from an existing store -// --------------------------------------------------------------------------- - -fn restore_gen(args: &[String]) -> Result<(), Failure> { - let store_dir = PathBuf::from(args.first().ok_or("restore-gen: missing ")?); - let volume_hex = args.get(1).ok_or("restore-gen: missing ")?; - let generation_hex = args.get(2).ok_or("restore-gen: missing ")?; - let destination = PathBuf::from(args.get(3).ok_or("restore-gen: missing ")?); - fs::create_dir_all(&destination)?; - - let volume_uuid: [u8; 16] = { - let clean: String = volume_hex.chars().filter(|c| *c != '-').collect(); - let bytes = hex_decode(&clean)?; - bytes - .as_slice() - .try_into() - .map_err(|_| "volume uuid must be 16 bytes")? - }; - let digest: [u8; 32] = hex_decode(generation_hex)? - .as_slice() - .try_into() - .map_err(|_| "generation must be 32 bytes")?; - - let runtime = tokio::runtime::Runtime::new()?; - runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .open_volume( - VolumeId::from_bytes(volume_uuid), - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("open volume"))? - .value; - let mut checkout = volume - .checkout( - GenerationSelector::Exact(GenerationId::new(acyclic_fs::Digest::from_bytes( - digest, - ))), - CheckoutMode { - access: AccessMode::ReadOnly, - consistency: ConsistencyMode::Pinned, - mutations: MutationMode::None, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout exact"))? - .value; - let receipt = materialize_checkout( - &mut checkout, - &MaterializeOptions { - destination: destination.clone(), - maximum_directory_entries: 1_024, - maximum_extent_spans: 65_536, - transfer_bytes: 8 * 1024 * 1024, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("materialize"))? - .value; - println!( - "materialized {} files / {} dirs into {}", - receipt.files, - receipt.directories, - destination.display() - ); - Ok(()) - }) -} - -fn hex_decode(text: &str) -> Result, Failure> { - if !text.len().is_multiple_of(2) { - return Err("odd hex length".into()); - } - (0..text.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&text[i..i + 2], 16).map_err(|e| format!("{e}").into())) - .collect() -} - -// --------------------------------------------------------------------------- -// mount-smoke: Launch 3 gate — writable native mount of a captured checkout -// --------------------------------------------------------------------------- - -#[allow( - clippy::too_many_lines, - reason = "the gate's mount, write, unmount, and verify steps are one straight-line procedure" -)] -fn mount_smoke(args: &[String]) -> Result<(), Failure> { - use acyclic_fs::{ - mount_native, probe_native_mount, CheckoutMountSource, NativeMountRequest, SharedCheckout, - }; - use std::sync::Arc; - - let source = PathBuf::from(args.first().ok_or("mount-smoke: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("mount-smoke: missing ")?); - let store_dir = work.join("store"); - let mount_dir = work.join("mnt"); - fs::create_dir_all(&store_dir)?; - fs::create_dir_all(&mount_dir)?; - - let capabilities = probe_native_mount(); - println!( - "probe: kind={:?} available={} writable={} reason={:?}", - capabilities.kind, - capabilities.available, - capabilities.writable, - capabilities.unavailable_reason - ); - if !capabilities.available || !capabilities.writable { - return Err("native mount unavailable or read-only — fork engine gate FAILS".into()); - } - - // Capture the fixture and publish it, exactly like the daemon does. - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, _generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - - // Fresh writable Head checkout for the mount (the fork shape). - let config = volume_config(); - let (checkout, fs_engine) = runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("reopen store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("open volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout head"))? - .value; - Ok::<_, Failure>((checkout, fs_engine)) - })?; - let _keep_engine_alive = fs_engine; - - let shared = Arc::new(SharedCheckout::new(checkout)); - let mount_source = Arc::new( - CheckoutMountSource::new(Arc::clone(&shared), config) - .map_err(engine_err("mount source"))?, - ); - let started = Instant::now(); - let mut session = mount_native( - NativeMountRequest { - mount_id: acyclic_fs::MountId::new(), - volume_id, - destination: mount_dir.clone(), - writable: true, - }, - mount_source, - ) - .map_err(engine_err("mount_native"))?; - println!( - "mounted at {} in {:?}", - mount_dir.display(), - started.elapsed() - ); - - // Everything below must not leave the mount attached on failure. - let verdict = (|| -> Result<(), Failure> { - // Read path: lazy listing + content identical to the fixture. - let started = Instant::now(); - let mounted_readme = fs::read(mount_dir.join("README.md"))?; - println!("first small read: {:?}", started.elapsed()); - if mounted_readme != fs::read(source.join("README.md"))? { - return Err("README content mismatch through mount".into()); - } - let started = Instant::now(); - let mounted_asset = fs::read(mount_dir.join("assets.bin"))?; - println!( - "8MB hydration read: {:?} ({} bytes)", - started.elapsed(), - mounted_asset.len() - ); - if mounted_asset != fs::read(source.join("assets.bin"))? { - return Err("asset content mismatch through mount".into()); - } - let mounted_link = fs::read_link(mount_dir.join("node_modules/.bin/tool"))?; - if mounted_link != Path::new("../../tool.sh") { - return Err(format!("symlink mismatch through mount: {mounted_link:?}").into()); - } - - // Write path: overlay writes visible in the mount, invisible outside. - fs::write(mount_dir.join("fork-note.txt"), b"written in the fork\n")?; - fs::write(mount_dir.join("README.md"), b"edited in the fork\n")?; - fs::create_dir(mount_dir.join("fork-dir"))?; - if fs::read(mount_dir.join("fork-note.txt"))? != b"written in the fork\n" { - return Err("write-then-read mismatch through mount".into()); - } - if fs::read(mount_dir.join("README.md"))? != b"edited in the fork\n" { - return Err("edit-then-read mismatch through mount".into()); - } - if source.join("fork-note.txt").exists() { - return Err("overlay write leaked into the source tree".into()); - } - if fs::read(source.join("README.md"))? != mounted_readme { - return Err("overlay edit leaked into the source tree".into()); - } - Ok(()) - })(); - - let stopped = session.stop(); - println!("unmounted cleanly: {stopped:?}"); - verdict?; - stopped.map_err(engine_err("unmount"))?; - - println!("MOUNT SMOKE OK: writable native mount serves, isolates, and detaches"); - Ok(()) -} - -// --------------------------------------------------------------------------- -// mount-hold: attach one mount and hold it for so failing -// operations can be probed interactively from a shell. -// --------------------------------------------------------------------------- - -fn mount_hold(args: &[String]) -> Result<(), Failure> { - use acyclic_fs::{mount_native, CheckoutMountSource, NativeMountRequest, SharedCheckout}; - use std::sync::Arc; - - let source = PathBuf::from(args.first().ok_or("mount-hold: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("mount-hold: missing ")?); - let seconds: u64 = args.get(2).map_or(Ok(60), |value| value.parse())?; - let store_dir = work.join("store"); - let mount_dir = work.join("mnt"); - fs::create_dir_all(&store_dir)?; - fs::create_dir_all(&mount_dir)?; - - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, _generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - let config = volume_config(); - let checkout = runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("open volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout"))? - .value; - std::mem::forget(fs_engine); - Ok::<_, Failure>(checkout) - })?; - let shared = Arc::new(SharedCheckout::new(checkout)); - let mount_source = - Arc::new(CheckoutMountSource::new(shared, config).map_err(engine_err("mount source"))?); - let mut session = mount_native( - NativeMountRequest { - mount_id: acyclic_fs::MountId::new(), - volume_id, - destination: mount_dir.clone(), - writable: true, - }, - mount_source, - ) - .map_err(engine_err("mount"))?; - println!("HELD: {} for {seconds}s", mount_dir.display()); - std::thread::sleep(std::time::Duration::from_secs(seconds)); - session.stop().map_err(engine_err("unmount"))?; - println!("released"); - Ok(()) -} - -// --------------------------------------------------------------------------- -// source-probe: exercise MountFilesystem directly (no kernel, no NFS) to -// pinpoint which callback fails and with what typed error. -// --------------------------------------------------------------------------- - -fn source_probe(args: &[String]) -> Result<(), Failure> { - use acyclic_fs::{CheckoutMountSource, MountFilesystem, MountPath, SharedCheckout}; - use std::sync::Arc; - - let source = - PathBuf::from(args.first().ok_or("source-probe: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("source-probe: missing ")?); - let store_dir = work.join("store"); - fs::create_dir_all(&store_dir)?; - - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, _generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - let config = volume_config(); - let checkout = runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("open volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout"))? - .value; - std::mem::forget(fs_engine); - Ok::<_, Failure>(checkout) - })?; - let shared = Arc::new(SharedCheckout::new(checkout)); - let mount_source = - CheckoutMountSource::new(shared, config).map_err(engine_err("mount source"))?; - - let readme = MountPath::root().child(b"README.md".to_vec()); - println!( - "lookup(/): {:?}", - mount_source - .lookup(&MountPath::root()) - .map(|l| l.map(|l| l.node.kind)) - ); - println!( - "lookup(README.md): {:?}", - mount_source - .lookup(&readme) - .map(|l| l.map(|l| (l.node.kind, l.node.logical_bytes))) - ); - println!( - "read_directory(/): {:?}", - mount_source - .read_directory(&MountPath::root(), None, 16) - .map(|p| p.entries.len()) - ); - match mount_source.open_file(&readme) { - Ok(file) => { - println!("open_file(README): Ok"); - println!( - " handle.lookup(): {:?}", - file.lookup().map(|l| l.node.logical_bytes) - ); - println!( - " read_range(0,13): {:?}", - file.read_range(0, 13) - .map(|b| String::from_utf8_lossy(&b).into_owned()) - ); - } - Err(error) => println!("open_file(README): ERR {error:?}"), - } - Ok(()) -} - -// --------------------------------------------------------------------------- -// mount-smoke2: TWO simultaneous native mounts from ONE process — the -// minimal repro for the fork engine's N>1 requirement. -// --------------------------------------------------------------------------- - -fn mount_smoke2(args: &[String]) -> Result<(), Failure> { - use acyclic_fs::{mount_native, CheckoutMountSource, NativeMountRequest, SharedCheckout}; - use std::sync::Arc; - - let source = - PathBuf::from(args.first().ok_or("mount-smoke2: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("mount-smoke2: missing ")?); - let store_dir = work.join("store"); - fs::create_dir_all(&store_dir)?; - - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, _generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - let config = volume_config(); - - let mut sessions = Vec::new(); - for index in 0..2u32 { - let mount_dir = work.join(format!("mnt{index}")); - fs::create_dir_all(&mount_dir)?; - let checkout = runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("open volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout"))? - .value; - std::mem::forget(fs_engine); // keep engine alive for the mount - Ok::<_, Failure>(checkout) - })?; - let shared = Arc::new(SharedCheckout::new(checkout)); - let mount_source = - Arc::new(CheckoutMountSource::new(shared, config).map_err(engine_err("mount source"))?); - let started = Instant::now(); - let session = mount_native( - NativeMountRequest { - mount_id: acyclic_fs::MountId::new(), - volume_id, - destination: mount_dir.clone(), - writable: true, - }, - mount_source, - ) - .map_err(engine_err(if index == 0 { - "FIRST mount" - } else { - "SECOND mount" - }))?; - println!( - "mount {index} attached at {} in {:?}", - mount_dir.display(), - started.elapsed() - ); - let listing = fs::read_dir(&mount_dir)?.count(); - println!("mount {index} lists {listing} entries"); - sessions.push(session); - } - - for (index, mut session) in sessions.into_iter().enumerate() { - session.stop().map_err(engine_err("unmount"))?; - println!("mount {index} detached"); - } - println!("MOUNT SMOKE2 OK: two simultaneous sessions in one process"); - Ok(()) -} - -// --------------------------------------------------------------------------- -// corpus: synthetic tree of files totalling MiB, 100 per dir -// --------------------------------------------------------------------------- - -fn corpus(args: &[String]) -> Result<(), Failure> { - let root = PathBuf::from(args.first().ok_or("corpus: missing ")?); - let files: u64 = args.get(1).ok_or("corpus: missing ")?.parse()?; - let total_mb: u64 = args.get(2).ok_or("corpus: missing ")?.parse()?; - let bytes_per_file = (total_mb * 1024 * 1024) / files.max(1); - let mut payload = Vec::with_capacity(bytes_per_file as usize); - let mut state: u32 = 0x1234_5678; - while (payload.len() as u64) < bytes_per_file { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - payload.extend_from_slice(&state.to_le_bytes()); - } - let started = Instant::now(); - for index in 0..files { - let dir = root.join(format!("dir-{:05}", index / 100)); - if index % 100 == 0 { - fs::create_dir_all(&dir)?; - } - fs::write(dir.join(format!("file-{index:07}.dat")), &payload)?; - } - println!( - "corpus: {files} files x {bytes_per_file} bytes in {:?}", - started.elapsed() - ); - Ok(()) -} - -// --------------------------------------------------------------------------- -// bench: baseline capture, then watch-driven incremental capture latency -// --------------------------------------------------------------------------- - -fn bench(args: &[String]) -> Result<(), Failure> { - let source = PathBuf::from(args.first().ok_or("bench: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("bench: missing ")?); - let rounds: usize = args.get(2).map_or(Ok(30), |value| value.parse())?; - let store_dir = work.join("store"); - fs::create_dir_all(&store_dir)?; - let runtime = tokio::runtime::Runtime::new()?; - runtime.block_on(bench_inner(&source, &store_dir, rounds)) -} - -#[allow( - clippy::too_many_lines, - reason = "the benchmark's setup, rounds, and report are one straight-line procedure" -)] -async fn bench_inner(source: &Path, store_dir: &Path, rounds: usize) -> Result<(), Failure> { - use acyclic_fs::model::VolumeLimits; - use acyclic_fs::{NativeWatch, NativeWatchOptions, WatchBatch}; - - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .create_volume(volume_config(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("create volume"))? - .value; - let mut checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout"))? - .value; - - let mut watch = NativeWatch::open( - source, - NativeWatchOptions { - limits: VolumeLimits::default(), - maximum_queued_changes: 65_536, - recursive: true, - }, - ) - .map_err(engine_err("watch open"))?; - watch.begin_rescan().map_err(engine_err("begin_rescan"))?; - - let options = CaptureOptions { - source_root: source.to_path_buf(), - expected_root_identity: capture_root_identity(source).map_err(engine_err("identity"))?, - maximum_paths: 4_000_000, - maximum_extent_spans: 65_536, - }; - - let started = Instant::now(); - let receipt = capture_baseline(&mut checkout, &options, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("baseline"))? - .value; - let baseline_capture = started.elapsed(); - let started = Instant::now(); - checkout - .commit(OperationId::new(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("baseline commit"))?; - println!( - "baseline: capture {:?} + commit {:?} ({} paths, {} bytes)", - baseline_capture, - started.elapsed(), - receipt.examined_paths, - receipt.staged_file_bytes - ); - watch.finish_rescan().map_err(engine_err("finish_rescan"))?; - - let mut event_latency = Vec::with_capacity(rounds); - let mut capture_only = Vec::with_capacity(rounds); - let mut capture_checkpoint = Vec::with_capacity(rounds); - let mut capture_commit = Vec::with_capacity(rounds); - for round in 0..rounds { - let target = source.join(format!("bench-mutation-{}.txt", round % 5)); - let mutated_at = Instant::now(); - fs::write(&target, format!("round {round} at {mutated_at:?}\n"))?; - - // Poll until the watcher reports the change (event-arrival latency). - let batch = loop { - let batch = watch - .poll(4_096, WorkCounters::UNBOUNDED, &cancel) - .map_err(engine_err("poll"))? - .value; - match &batch { - WatchBatch::Changes { changes, .. } if !changes.is_empty() => break batch, - WatchBatch::Changes { .. } => { - std::thread::sleep(std::time::Duration::from_millis(2)); - } - WatchBatch::RescanRequired { reason, .. } => { - return Err(format!("rescan required mid-bench: {reason:?}").into()); - } - } - }; - event_latency.push(mutated_at.elapsed()); - - let capture_started = Instant::now(); - acyclic_fs::capture_watch_batch( - &mut checkout, - batch, - &options, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("capture_watch_batch"))?; - let captured_at = capture_started.elapsed(); - if round % 2 == 0 { - checkout - .checkpoint(WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("incremental checkpoint"))?; - capture_checkpoint.push(capture_started.elapsed()); - } else { - checkout - .commit(OperationId::new(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("incremental commit"))?; - capture_commit.push(capture_started.elapsed()); - } - capture_only.push(captured_at); - } - - report("event-arrival latency", &mut event_latency); - report("capture (no publish) latency", &mut capture_only); - report("capture+checkpoint latency", &mut capture_checkpoint); - report("capture+commit latency", &mut capture_commit); - Ok(()) -} - -fn report(label: &str, samples: &mut [std::time::Duration]) { - samples.sort(); - let p = |q: f64| samples[((samples.len() - 1) as f64 * q) as usize]; - println!( - "{label}: n={} p50={:?} p95={:?} max={:?}", - samples.len(), - p(0.50), - p(0.95), - samples[samples.len() - 1] - ); -} - -// --------------------------------------------------------------------------- -// roundtrip -// --------------------------------------------------------------------------- - -fn roundtrip(args: &[String]) -> Result<(), Failure> { - let source = PathBuf::from(args.first().ok_or("roundtrip: missing ")?) - .canonicalize() - .map_err(engine_err("canonicalize "))?; - let work = PathBuf::from(args.get(1).ok_or("roundtrip: missing ")?); - let store_dir = work.join("store"); - let restore_dir = work.join("restore"); - fs::create_dir_all(&store_dir)?; - fs::create_dir_all(&restore_dir)?; - if fs::read_dir(&restore_dir)?.next().is_some() { - return Err("roundtrip: /restore must be empty".into()); - } - - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - runtime.block_on(materialize(&store_dir, volume_id, generation, &restore_dir))?; - - let started = Instant::now(); - let mismatches = compare_trees(&source, &restore_dir)?; - println!("compare: {:?}", started.elapsed()); - - if mismatches.is_empty() { - println!("ROUNDTRIP OK: content + mode identical"); - Ok(()) - } else { - for m in &mismatches { - eprintln!("MISMATCH: {m}"); - } - Err(format!("{} mismatches", mismatches.len()).into()) - } -} - -fn volume_config() -> VolumeConfig { - // QUAL_PROFILE=portable switches the profile for differential debugging. - let profile = match std::env::var("QUAL_PROFILE").as_deref() { - Ok("portable") => FilesystemProfile::Portable, - _ => FilesystemProfile::Posix, - }; - let mut config = VolumeConfig { - profile, - ..VolumeConfig::portable(Lifecycle::Durable) - }; - // A baseline capture is one atomic authored transaction covering every - // path in the repo; the default 2,048-mutation batch cap rejects any - // real tree. Sized for large monorepos. - config.limits.maximum_mutations_per_batch = 4_194_304; - config.limits.maximum_paths_per_batch = 4_194_304; - config -} - -async fn capture_and_commit( - source: &Path, - store_dir: &Path, -) -> Result<(VolumeId, GenerationId), Failure> { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(store_dir)) - .await - .map_err(engine_err("open store"))?; - - let started = Instant::now(); - let volume = fs_engine - .create_volume(volume_config(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("create volume"))? - .value; - let mut checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout head"))? - .value; - println!("volume + checkout: {:?}", started.elapsed()); - - let options = CaptureOptions { - source_root: source.to_path_buf(), - expected_root_identity: capture_root_identity(source) - .map_err(engine_err("root identity"))?, - maximum_paths: 4_000_000, - maximum_extent_spans: 65_536, - }; - let started = Instant::now(); - let receipt = capture_baseline(&mut checkout, &options, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("capture_baseline"))? - .value; - println!( - "capture_baseline: {:?} (examined {} paths, {} changed, {} bytes staged)", - started.elapsed(), - receipt.examined_paths, - receipt.changed_paths, - receipt.staged_file_bytes - ); - - let started = Instant::now(); - if std::env::var_os("QUAL_CHECKPOINT_ONLY").is_some() { - let generation = checkout - .checkpoint(WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("checkpoint"))? - .value; - println!("checkpoint (no publish): {:?}", started.elapsed()); - return Ok((checkout.volume_id(), generation)); - } - let outcome = checkout - .commit(OperationId::new(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("commit"))? - .value; - println!("commit: {:?}", started.elapsed()); - match outcome { - CheckoutCommitOutcome::Committed { generation_id, .. } - | CheckoutCommitOutcome::AlreadyCommitted { generation_id, .. } => { - Ok((checkout.volume_id(), generation_id)) - } - other => Err(format!("commit not durable: {other:?}").into()), - } -} - -async fn materialize( - store_dir: &Path, - volume_id: VolumeId, - generation: GenerationId, - destination: &Path, -) -> Result<(), Failure> { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(store_dir)) - .await - .map_err(engine_err("reopen store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("reopen volume"))? - .value; - let mut checkout = volume - .checkout( - GenerationSelector::Exact(generation), - CheckoutMode { - access: AccessMode::ReadOnly, - consistency: ConsistencyMode::Pinned, - mutations: MutationMode::None, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout exact"))? - .value; - - let started = Instant::now(); - let receipt = materialize_checkout( - &mut checkout, - &MaterializeOptions { - destination: destination.to_path_buf(), - maximum_directory_entries: 1_024, - maximum_extent_spans: 65_536, - transfer_bytes: 8 * 1024 * 1024, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("materialize_checkout"))? - .value; - println!( - "materialize: {:?} ({} files, {} dirs, {} symlinks, {} bytes written)", - started.elapsed(), - receipt.files, - receipt.directories, - receipt.symbolic_links, - receipt.written_bytes - ); - Ok(()) -} - -// --------------------------------------------------------------------------- -// tree comparison: content + kind + symlink target + mode bits (no mtime/uid) -// --------------------------------------------------------------------------- - -#[derive(Debug, PartialEq)] -enum EntryKind { - File, - Dir, - Symlink, - Other, -} - -fn walk(root: &Path) -> Result, Failure> { - let mut entries = BTreeMap::new(); - let mut stack = vec![root.to_path_buf()]; - while let Some(dir) = stack.pop() { - for entry in fs::read_dir(&dir)? { - let path = entry?.path(); - let meta = fs::symlink_metadata(&path)?; - let relative = path.strip_prefix(root)?.to_path_buf(); - let kind = if meta.file_type().is_symlink() { - EntryKind::Symlink - } else if meta.is_dir() { - stack.push(path.clone()); - EntryKind::Dir - } else if meta.is_file() { - EntryKind::File - } else { - EntryKind::Other - }; - entries.insert(relative, kind); - } - } - Ok(entries) -} - -fn compare_trees(source: &Path, restored: &Path) -> Result, Failure> { - let left = walk(source)?; - let right = walk(restored)?; - let mut mismatches = Vec::new(); - - for (path, kind) in &left { - match right.get(path) { - None => mismatches.push(format!("{}: missing in restore", path.display())), - Some(other) if other != kind => { - mismatches.push(format!("{}: kind {kind:?} vs {other:?}", path.display())); - } - Some(_) => { - let a = source.join(path); - let b = restored.join(path); - match kind { - EntryKind::Symlink => { - let ta = fs::read_link(&a)?; - let tb = fs::read_link(&b)?; - if ta != tb { - mismatches.push(format!( - "{}: symlink target {:?} vs {:?}", - path.display(), - ta, - tb - )); - } - } - EntryKind::File => { - if !files_equal(&a, &b)? { - mismatches.push(format!("{}: content differs", path.display())); - } - if let Some(m) = mode_mismatch(&a, &b, path)? { - mismatches.push(m); - } - } - EntryKind::Dir => { - if let Some(m) = mode_mismatch(&a, &b, path)? { - mismatches.push(m); - } - } - EntryKind::Other => {} - } - } - } - } - for path in right.keys() { - if !left.contains_key(path) { - mismatches.push(format!("{}: extra in restore", path.display())); - } - } - Ok(mismatches) -} - -fn files_equal(a: &Path, b: &Path) -> Result { - let (ma, mb) = (fs::metadata(a)?, fs::metadata(b)?); - if ma.len() != mb.len() { - return Ok(false); - } - let (mut fa, mut fb) = (fs::File::open(a)?, fs::File::open(b)?); - let mut ba = vec![0u8; 1024 * 1024]; - let mut bb = vec![0u8; 1024 * 1024]; - loop { - let na = fa.read(&mut ba)?; - let nb = fb.read(&mut bb)?; - if na != nb || ba[..na] != bb[..nb] { - return Ok(false); - } - if na == 0 { - return Ok(true); - } - } -} - -#[cfg(unix)] -fn mode_mismatch(a: &Path, b: &Path, relative: &Path) -> Result, Failure> { - use std::os::unix::fs::MetadataExt; - let ma = fs::metadata(a)?.mode() & 0o7777; - let mb = fs::metadata(b)?.mode() & 0o7777; - if ma == mb { - Ok(None) - } else { - Ok(Some(format!( - "{}: mode {ma:o} vs {mb:o}", - relative.display() - ))) - } -} - -#[cfg(not(unix))] -fn mode_mismatch(_a: &Path, _b: &Path, _relative: &Path) -> Result, Failure> { - Ok(None) -} diff --git a/scripts/check-product-name.sh b/scripts/check-product-name.sh index 81858fb..ba45309 100755 --- a/scripts/check-product-name.sh +++ b/scripts/check-product-name.sh @@ -3,7 +3,7 @@ # 1. scripts/install.sh is fetched standalone and mirrors `name`, # `github_repo`, and `npm_package`; they must match exactly. # 2. No user-facing Rust source spells the name out. Only crate/module -# identifiers (acyclic_fs, acyclic-qual, ...) and comments +# identifiers (acyclic_fs, ...) and comments # may contain it; strings, paths, and doc templates go through # `product::NAME` / `product::render`. set -euo pipefail diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 1b56fbc..616c1a0 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -1,78 +1,6 @@ #!/usr/bin/env bash -# Runs what .github/workflows/ci.yml runs, job by job, on this machine, and -# prints one line per step at the end so a red step is easy to find. Every -# step runs even after an earlier one fails (CI's jobs are independent too); -# the exit code is non-zero if any step failed. -# -# scripts/ci-local.sh everything, including the acceptance suite (slowest) -# scripts/ci-local.sh --no-acceptance the static gates, unit tests, release build, coverage -# scripts/ci-local.sh --no-coverage skip cargo-llvm-cov (needs `cargo install cargo-llvm-cov`) -# -# Needs: stable toolchain with rustfmt + clippy, cargo-deny, node (for the -# duplication check), and unless --no-coverage: cargo-llvm-cov plus the -# `llvm-tools-preview` rustup component it drives -# (`rustup component add llvm-tools-preview`). macOS also needs -# FUSE-T for the fork/Safe Mode acceptance scripts. -set -uo pipefail +# The same bounded gate runs locally and on each supported CI host. +set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" -# Same as CI: the pinned acyclic-fs git dependency needs the git CLI's -# credentials and protocol support, not cargo's built-in fetcher. -export CARGO_NET_GIT_FETCH_WITH_CLI=true - -run_acceptance=1 -run_coverage=1 -for arg in "$@"; do - case "$arg" in - --no-acceptance) run_acceptance=0 ;; - --no-coverage) run_coverage=0 ;; - *) echo "unknown flag: $arg" >&2; exit 2 ;; - esac -done - -results=() -failed=0 -step() { - local name="$1" - shift - echo - echo "=== $name" - local started=$SECONDS - if "$@"; then - results+=("PASS $((SECONDS - started))s $name") - else - results+=("FAIL $((SECONDS - started))s $name") - failed=1 - fi -} - -# deny job -step "product name single-sourced" bash scripts/check-product-name.sh -step "no secrets or forbidden files" bash scripts/check-no-secrets.sh -step "code quality (width, TODOs, comment blocks, duplication)" bash scripts/check-code-quality.sh -step "cargo deny" cargo deny --locked check - -# lint job -step "cargo fmt --check" cargo fmt --all --check -step "cargo clippy -D warnings" cargo clippy --workspace --all-targets --all-features -- -D warnings - -# test job -step "cargo test" cargo test --workspace -step "cargo build --release" cargo build --release -if [ "$run_acceptance" -eq 1 ]; then - step "acceptance suite" env \ - ACYCLIC_BIN="$ROOT/target/release/acyclic" \ - ACYCLIC_QUAL="$ROOT/target/release/acyclic-qual" \ - ACYCLIC_LAT_FILES=5000 ACYCLIC_LAT_MB=64 ACYCLIC_SOAK_ROUNDS=30 \ - bash tests/acceptance/run-all.sh -fi - -# coverage job -if [ "$run_coverage" -eq 1 ]; then - step "coverage (floor: see ci.yml)" cargo llvm-cov --workspace --all-features --summary-only --fail-under-lines 48 -fi - -echo -echo "=== summary" -printf '%s\n' "${results[@]}" -exit "$failed" +SDK="$(cd "$ROOT/../../../sdk" && pwd)" +exec python3 "$SDK/scripts/qualify-local.py" --plugin-root "$ROOT" "$@"\n \ No newline at end of file diff --git a/scripts/docker-linux.sh b/scripts/docker-linux.sh index 8a534d2..f87c910 100755 --- a/scripts/docker-linux.sh +++ b/scripts/docker-linux.sh @@ -1,51 +1,24 @@ #!/usr/bin/env bash -# Linux validation without leaving the Mac: build and run the full test + -# acceptance suite inside a Linux container. acyclic-fs is fetched from its -# own public git repo (see Cargo.toml), not a sibling checkout; all build -# artifacts and test state stay on container-local filesystems (which is -# also what exercises renameat2(RENAME_EXCHANGE) and inotify on a Linux -# kernel for real). -# -# Usage: scripts/docker-linux.sh [image] +# Run the identical qualification gate on a Linux kernel in a container. set -euo pipefail - PLUGIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SDK="$(cd "$PLUGIN/../../../sdk" && pwd)" IMAGE="${1:-rust:1-bookworm}" - -# FUSE inside the container (fork mounts): pass the device + cap when the -# host offers them; forks.sh skips gracefully otherwise. FUSE_FLAGS=() -if [ -e /dev/fuse ] || [ "$(uname -s)" = "Darwin" ]; then +if [ -e /dev/fuse ]; then FUSE_FLAGS=(--device /dev/fuse --cap-add SYS_ADMIN) fi - docker run --rm \ "${FUSE_FLAGS[@]}" \ - -v "$PLUGIN:/src/graphcoder-plugin:ro" \ + -v "$PLUGIN:/lab/worktrees/graphcoder/plugin-sdk-lab:ro" \ + -v "$SDK:/lab/sdk:ro" \ -v acyclic-linux-cargo:/cargo \ -v acyclic-linux-target:/build \ -e CARGO_HOME=/cargo \ -e CARGO_TARGET_DIR=/build/target \ - -e CARGO_NET_GIT_FETCH_WITH_CLI=true \ "$IMAGE" bash -eu -o pipefail -c ' export DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null apt-get install -y -qq sqlite3 procps python3 >/dev/null - - cd /src/graphcoder-plugin - echo "=== cargo test (workspace)" - cargo test --workspace 2>&1 | grep -E "test result|error" || true - cargo test --workspace >/dev/null - - echo "=== release build" - cargo build --release - - echo "=== acceptance suite" - export TMPDIR=/tmp - export ACYCLIC_BIN=/build/target/release/acyclic - export ACYCLIC_QUAL=/build/target/release/acyclic-qual - export ACYCLIC_LAT_FILES=5000 - export ACYCLIC_LAT_MB=64 - export ACYCLIC_SOAK_ROUNDS=30 - bash tests/acceptance/run-all.sh - ' + python3 /lab/sdk/scripts/qualify-local.py + '\n \ No newline at end of file diff --git a/tests/acceptance/common.sh b/tests/acceptance/common.sh index f5d1d2c..25bb4c6 100755 --- a/tests/acceptance/common.sh +++ b/tests/acceptance/common.sh @@ -5,7 +5,8 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" BIN="${ACYCLIC_BIN:-$REPO_ROOT/target/debug/acyclic}" -QUAL="${ACYCLIC_QUAL:-$REPO_ROOT/target/debug/acyclic-qual}" +SDK_ROOT="$(cd "$REPO_ROOT/../../../sdk" && pwd)" +QUAL="${ACYCLIC_QUAL:-$SDK_ROOT/target/debug/qualify}" WORK="$(mktemp -d "${TMPDIR:-/tmp}/acyclic-acceptance.XXXXXX")" # Canonicalize: macOS TMPDIR ends in "/" and /var -> /private/var, so the diff --git a/tests/acceptance/run-all.sh b/tests/acceptance/run-all.sh index dda5eb9..f6be81e 100755 --- a/tests/acceptance/run-all.sh +++ b/tests/acceptance/run-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Runs the full acceptance suite against the built binaries. -# ACYCLIC_BIN / ACYCLIC_QUAL override binary paths (default: target/debug) +# ACYCLIC_BIN / ACYCLIC_QUAL override plugin and SDK qualifier paths # Individual scripts also run standalone. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" From 47f442cc7a73e07ee860d06a6440a1de9e3dd653 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 10:42:14 +0100 Subject: [PATCH 010/106] Fix shared gate wrapper commands --- .github/workflows/ci.yml | 2 +- scripts/ci-local.sh | 2 +- scripts/docker-linux.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66ea622..101c2ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,4 +53,4 @@ jobs: qualify-${{ matrix.os }}- - name: Run the shared bounded gate shell: pwsh - run: python sdk/scripts/qualify-local.py --plugin-root worktrees/graphcoder/plugin-sdk-lab\n \ No newline at end of file + run: python sdk/scripts/qualify-local.py --plugin-root worktrees/graphcoder/plugin-sdk-lab diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 616c1a0..ab39c24 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -3,4 +3,4 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SDK="$(cd "$ROOT/../../../sdk" && pwd)" -exec python3 "$SDK/scripts/qualify-local.py" --plugin-root "$ROOT" "$@"\n \ No newline at end of file +exec python3 "$SDK/scripts/qualify-local.py" --plugin-root "$ROOT" "$@" diff --git a/scripts/docker-linux.sh b/scripts/docker-linux.sh index f87c910..08ba569 100755 --- a/scripts/docker-linux.sh +++ b/scripts/docker-linux.sh @@ -21,4 +21,4 @@ docker run --rm \ apt-get update -qq >/dev/null apt-get install -y -qq sqlite3 procps python3 >/dev/null python3 /lab/sdk/scripts/qualify-local.py - '\n \ No newline at end of file + ' From a4a57020a6167c09279c668dbf34a9dbd32be10d Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 11:42:28 +0100 Subject: [PATCH 011/106] Reject plugin stores inside repository trees --- crates/acyclic/src/server.rs | 2 +- crates/acyclic/src/store.rs | 169 +++++++++++++++++++++++++++++-- crates/acyclic/tests/pipeline.rs | 4 +- 3 files changed, 165 insertions(+), 10 deletions(-) diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 809d949..b64cd4e 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -215,7 +215,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { let listener = bind_socket(&runtime, &paths)?; lap("socket bind + pidfile"); let store = runtime - .block_on(Store::open(paths.clone())) + .block_on(Store::open(repo_root, paths.clone())) .map_err(|error| error.to_string())?; let repo_root = store.repo_root.clone(); lap("store open"); diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index ebd2df0..ab6d6c8 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -37,22 +37,54 @@ impl StorePaths { /// Resolves the store root for a repo. `stores_root` override comes from /// config; the default is `~/.local/share/acyclic/stores`. pub fn for_repo(repo_root: &Path, stores_root: Option<&Path>) -> Result { + let canonical = repo_root + .canonicalize() + .map_err(|error| EngineError::Store(format!("canonicalize repo root: {error}")))?; let base = if let Some(path) = stores_root { - path.to_path_buf() + if path.is_absolute() { + path.to_path_buf() + } else { + canonical.join(path) + } } else { let home = std::env::var_os("HOME") .ok_or_else(|| EngineError::Store("HOME is not set".into()))?; Path::new(&home).join(format!(".local/share/{}/stores", crate::product::NAME)) }; - let canonical = repo_root - .canonicalize() - .map_err(|error| EngineError::Store(format!("canonicalize repo root: {error}")))?; + let base = canonicalize_planned(&base)?; let digest = blake3::hash(canonical.as_os_str().as_encoded_bytes()); let hex = digest.to_hex(); let short = hex.get(..16).unwrap_or(&hex); - Ok(Self { + let paths = Self { root: base.join(short), - }) + }; + paths.ensure_outside_repo(&canonical)?; + Ok(paths) + } + + fn ensure_outside_repo(&self, repo_root: &Path) -> Result<()> { + let repo = repo_root.canonicalize()?; + for path in [ + self.root.clone(), + self.object_store(), + self.index_db(), + self.spec_db(), + self.spec_runs(), + self.pidfile(), + self.rewind_journal(), + self.trash(), + self.meta(), + self.root.join("daemon.log"), + ] { + let resolved = canonicalize_planned(&path)?; + if resolved.starts_with(&repo) { + return Err(EngineError::Store(format!( + "store must be outside the repository: {}", + resolved.display() + ))); + } + } + Ok(()) } pub fn object_store(&self) -> PathBuf { @@ -96,6 +128,35 @@ impl StorePaths { } } +fn canonicalize_planned(path: &Path) -> std::io::Result { + let mut cursor = path; + let mut missing = Vec::new(); + loop { + match cursor.canonicalize() { + Ok(mut resolved) => { + for name in missing.into_iter().rev() { + resolved.push(name); + } + return Ok(resolved); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if cursor + .symlink_metadata() + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(std::io::Error::other("dangling store path symlink")); + } + let name = cursor.file_name().ok_or(error)?; + missing.push(name.to_os_string()); + cursor = cursor + .parent() + .ok_or_else(|| std::io::Error::other("store path has no existing ancestor"))?; + } + Err(error) => return Err(error), + } + } +} + /// Persisted store identity. The `VolumeId` MUST survive restarts: /// generations only resolve on their own volume. #[derive(Debug, Serialize, Deserialize)] @@ -165,14 +226,18 @@ impl Store { /// Creates the store for a repo: directories, volume, meta record. /// Fails if the store already exists. pub async fn init(repo_root: &Path, paths: StorePaths) -> Result { + paths.ensure_outside_repo(repo_root)?; if paths.meta().exists() { return Err(EngineError::Store(format!( "store already initialized at {}", paths.root.display() ))); } + std::fs::create_dir_all(&paths.root)?; + paths.ensure_outside_repo(repo_root)?; std::fs::create_dir_all(paths.object_store())?; std::fs::create_dir_all(paths.trash())?; + paths.ensure_outside_repo(repo_root)?; let cancel = CancellationToken::new(); let fs = LocalFs::local(local_options(paths.object_store())) @@ -213,7 +278,8 @@ impl Store { } /// Opens an existing store recorded in `meta.json`. - pub async fn open(paths: StorePaths) -> Result { + pub async fn open(repo_root: &Path, paths: StorePaths) -> Result { + paths.ensure_outside_repo(repo_root)?; let text = std::fs::read_to_string(paths.meta()).map_err(|error| { EngineError::Store(format!( "no store at {} ({error}); run init first", @@ -228,6 +294,13 @@ impl Store { meta.schema ))); } + let expected_repo = repo_root.canonicalize()?; + if meta.repo_root.canonicalize()? != expected_repo { + return Err(EngineError::Store( + "store belongs to a different repository".into(), + )); + } + paths.ensure_outside_repo(&expected_repo)?; let cancel = CancellationToken::new(); let phase = std::time::Instant::now(); @@ -336,7 +409,7 @@ mod tests { let created_id = created.volume_id; drop(created); - let reopened = Store::open(paths).await.expect("open"); + let reopened = Store::open(repo.path(), paths).await.expect("open"); assert_eq!(reopened.volume_id, created_id); assert_eq!( reopened.repo_root, @@ -352,4 +425,84 @@ mod tests { Store::init(repo.path(), paths.clone()).await.expect("init"); assert!(Store::init(repo.path(), paths).await.is_err()); } + + #[tokio::test] + async fn store_inside_repo_is_rejected_on_init_and_open() { + let repo = tempfile::tempdir().expect("repo dir"); + let paths = StorePaths { + root: repo.path().join(".stores").join("fixture"), + }; + assert!(StorePaths::for_repo(repo.path(), Some(&repo.path().join(".stores"))).is_err()); + assert!(matches!( + Store::init(repo.path(), paths.clone()).await, + Err(EngineError::Store(message)) if message.contains("outside the repository") + )); + assert!(!paths.root.exists()); + assert!(!paths.meta().exists()); + + std::fs::create_dir_all(&paths.root).expect("create invalid root"); + atomic_write_json( + &paths.meta(), + &StoreMeta { + schema: 1, + repo_root: repo.path().canonicalize().expect("canonical repo"), + volume_id: VolumeId::new(), + }, + ) + .expect("seed meta"); + assert!(matches!( + Store::open(repo.path(), paths).await, + Err(EngineError::Store(message)) if message.contains("outside the repository") + )); + } + + #[test] + fn relative_store_dir_resolves_from_repo_before_daemon_changes_cwd() { + let repo = tempfile::tempdir().expect("repo dir"); + let paths = StorePaths::for_repo(repo.path(), Some(Path::new("../stores"))) + .expect("resolve relative store"); + assert!(paths.root.is_absolute()); + assert!(paths.root.starts_with( + repo.path() + .canonicalize() + .expect("canonical repo") + .parent() + .expect("repo parent") + .join("stores") + )); + } + + #[tokio::test] + async fn store_open_rejects_a_different_requested_repository() { + let repo = tempfile::tempdir().expect("repo dir"); + let other = tempfile::tempdir().expect("other repo dir"); + let stores = tempfile::tempdir().expect("stores dir"); + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("resolve paths"); + drop(Store::init(repo.path(), paths.clone()).await.expect("init")); + assert!(matches!( + Store::open(other.path(), paths).await, + Err(EngineError::Store(message)) if message.contains("different repository") + )); + } + + #[cfg(unix)] + #[tokio::test] + async fn store_child_symlink_into_repo_is_rejected() { + let repo = tempfile::tempdir().expect("repo dir"); + let stores = tempfile::tempdir().expect("stores dir"); + let target = repo.path().join("misplaced-store"); + std::fs::create_dir(&target).expect("target dir"); + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("resolve paths"); + std::fs::create_dir_all(&paths.root).expect("store root"); + std::os::unix::fs::symlink(&target, paths.object_store()).expect("store link"); + assert!(StorePaths::for_repo(repo.path(), Some(stores.path())).is_err()); + assert!(matches!( + Store::init(repo.path(), paths).await, + Err(EngineError::Store(message)) if message.contains("outside the repository") + )); + assert_eq!( + std::fs::read_dir(target).expect("target entries").count(), + 0 + ); + } } diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index e59bcc5..bec18c8 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -135,7 +135,9 @@ fn checkpoint_rewind_journey() { thread.join().expect("pipeline thread"); // Diff runs against the reopened store (no daemon needed). - let store = runtime.block_on(Store::open(paths)).expect("reopen"); + let store = runtime + .block_on(Store::open(repo.path(), paths)) + .expect("reopen"); let changes = runtime .block_on(diff::diff(&store, pre_generation, post_generation)) .expect("diff"); From 4619f1bf1d2e7601ed26ab7efb6bee4ec36091d2 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 12:33:59 +0100 Subject: [PATCH 012/106] Refresh local SDK dependency lock for Windows publication --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 9ff3a3b..3fdcc3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,7 @@ dependencies = [ "prost-types", "thiserror", "tokio", + "windows-sys 0.61.2", ] [[package]] From b2e1e0173c3d3c5f6b94dfe6c38f06b27e7e19c9 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 14:09:07 +0100 Subject: [PATCH 013/106] Stop old watcher before intentional root swaps --- crates/acyclic/src/pipeline.rs | 80 ++++++++++++++++---------------- crates/acyclic/tests/fork.rs | 21 ++++++++- crates/acyclic/tests/pipeline.rs | 8 ++++ 3 files changed, 67 insertions(+), 42 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 93fd63a..8e86d9e 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -768,7 +768,7 @@ struct Pipeline { store: Store, index: Index, config: Config, - watch: NativeWatch, + watch: Option, options: CaptureOptions, /// Paths that never enter a checkpoint (`exclude` in the config). exclusions: Exclusions, @@ -913,7 +913,7 @@ impl Pipeline { store, index, config, - watch, + watch: Some(watch), options, exclusions, cancel, @@ -969,6 +969,8 @@ impl Pipeline { // batch; fold them in before declaring Ready. let batch = self .watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? .finish_rescan() .map_err(EngineError::fs("finish rescan"))?; // A root hint here is already covered by the rescan that just ran. @@ -1415,6 +1417,8 @@ impl Pipeline { polls += 1; let batch = self .watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? .poll(POLL_CHANGES, WorkCounters::UNBOUNDED, &self.cancel) .map_err(EngineError::fs("watch poll"))? .value; @@ -1685,20 +1689,7 @@ impl Pipeline { self.last_checkpoint_row = Some(safety_row); self.commit_engine().await?; - let outcome = rewind::execute( - &self.store, - target.generation, - self.config.trash_ttl_days, - &self.exclusions, - ) - .await; - - // The swap replaced the repo directory's inode: the pinned root - // identity and the watcher both point at the old tree. Rebuild both, - // then re-baseline. - self.reset_watch().await?; - self.baseline(CheckpointKind::Recovered).await?; - outcome + self.swap_root_and_rebaseline(target.generation).await } /// Single-path restore, bracketed by checkpoints: a `manual` safety row @@ -2097,16 +2088,7 @@ impl Pipeline { self.last_generation = generation; self.last_checkpoint_row = Some(row); - let swap = rewind::execute( - &self.store, - generation, - self.config.trash_ttl_days, - &self.exclusions, - ) - .await; - self.reset_watch().await?; - self.baseline(CheckpointKind::Recovered).await?; - let swap = swap?; + let swap = self.swap_root_and_rebaseline(generation).await?; Ok(PromoteOutcome::Promoted { generation, old_tree: Some(swap.old_tree), @@ -2137,6 +2119,8 @@ impl Pipeline { self.reset_watch().await?; let _ = self .watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? .finish_rescan() .map_err(EngineError::fs("finish rescan"))?; self.state = State::Ready; @@ -2238,7 +2222,21 @@ impl Pipeline { self.last_generation = generation; self.last_checkpoint_row = Some(row); - let swap = rewind::execute( + let swap = self.swap_root_and_rebaseline(generation).await?; + Ok(PromoteOutcome::Promoted { + generation, + old_tree: Some(swap.old_tree), + }) + } + + /// Stops the old event source before the intentional root exchange. An + /// old callback must never publish a hint into the new watcher's epoch. + async fn swap_root_and_rebaseline( + &mut self, + generation: GenerationId, + ) -> Result { + drop(self.watch.take()); + let outcome = rewind::execute( &self.store, generation, self.config.trash_ttl_days, @@ -2247,11 +2245,7 @@ impl Pipeline { .await; self.reset_watch().await?; self.baseline(CheckpointKind::Recovered).await?; - let swap = swap?; - Ok(PromoteOutcome::Promoted { - generation, - old_tree: Some(swap.old_tree), - }) + outcome } /// Reopens the watcher and recomputes the capture root identity — needed @@ -2264,16 +2258,20 @@ impl Pipeline { let repo_root = self.store.repo_root.clone(); self.options.expected_root_identity = capture_root_identity(&repo_root).map_err(EngineError::fs("root identity"))?; - self.watch = NativeWatch::open( - &repo_root, - NativeWatchOptions { - limits: VolumeLimits::default(), - maximum_queued_changes: WATCH_QUEUE, - recursive: true, - }, - ) - .map_err(EngineError::fs("reopen watcher"))?; + self.watch = Some( + NativeWatch::open( + &repo_root, + NativeWatchOptions { + limits: VolumeLimits::default(), + maximum_queued_changes: WATCH_QUEUE, + recursive: true, + }, + ) + .map_err(EngineError::fs("reopen watcher"))?, + ); self.watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? .begin_rescan() .map_err(EngineError::fs("begin rescan"))?; Ok(()) diff --git a/crates/acyclic/tests/fork.rs b/crates/acyclic/tests/fork.rs index 208338e..ab0e815 100644 --- a/crates/acyclic/tests/fork.rs +++ b/crates/acyclic/tests/fork.rs @@ -17,7 +17,7 @@ use std::time::Duration; use acyclic::config::Config; use acyclic::fork::{PromoteOutcome, SessionResolveOutcome}; use acyclic::index::{Attribution, CheckpointKind, Index}; -use acyclic::pipeline::{self, PipelineHandle}; +use acyclic::pipeline::{self, PipelineHandle, State}; use acyclic::store::{Store, StorePaths}; use acyclic_fs::kernel::{LogicalName, NamespacePath}; use acyclic_fs::model::VolumeLimits; @@ -118,6 +118,23 @@ async fn write_in_fork(seed: &acyclic::fork::ForkSeed, path: &str, bytes: &[u8]) .expect("create file in fork overlay"); } +fn assert_watcher_recovers_after_swap(rig: &Rig) { + rig.runtime.block_on(async { + rig.handle + .checkpoint(CheckpointKind::Post, Attribution::default()) + .await + .expect("checkpoint after root swap"); + let status = rig.handle.status().await.expect("status after root swap"); + assert_eq!(status.state, State::Ready, "{status:?}"); + #[cfg(not(target_os = "macos"))] + assert_eq!(status.watcher.invalidations, 0, "{status:?}"); + // FSEvents can replay a root move after the new watcher opens. A + // bounded recovery scan is safe; discarding that hint is not. + #[cfg(target_os = "macos")] + assert!(status.watcher.invalidations <= 1, "{status:?}"); + }); +} + /// P1 + P5: promote lands the fork's exact content and records attribution. #[test] fn promote_lands_fork_changes() { @@ -162,6 +179,7 @@ fn promote_lands_fork_changes() { assert!(rows.iter().any(|row| { row.kind == CheckpointKind::Manual && row.label.as_deref() == Some("promote test-fork") })); + assert_watcher_recovers_after_swap(&rig); rig.finish(); } @@ -207,6 +225,7 @@ fn resolve_then_apply_session_lands_fork_changes() { std::fs::read(rig.repo_path().join("fork-note.txt")).expect("landed file"), b"written in fork\n" ); + assert_watcher_recovers_after_swap(&rig); rig.finish(); } diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index bec18c8..1df5441 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -127,8 +127,16 @@ fn checkpoint_rewind_journey() { ); // Diff before → after names exactly the changed paths. + handle + .checkpoint(CheckpointKind::Post, Attribution::default()) + .await + .expect("checkpoint after rewind"); let status = handle.status().await.expect("status"); assert_eq!(status.state, pipeline::State::Ready); + #[cfg(not(target_os = "macos"))] + assert_eq!(status.watcher.invalidations, 0, "{status:?}"); + #[cfg(target_os = "macos")] + assert!(status.watcher.invalidations <= 1, "{status:?}"); handle.shutdown().await.expect("shutdown"); (before.generation, after.generation) }); From 3a8f9f24d5b3032c784e7379a59756d7409adc33 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 14:14:43 +0100 Subject: [PATCH 014/106] Assert Windows baseline readiness without polling sleep --- tests/acceptance/windows-smoke.sh | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/acceptance/windows-smoke.sh b/tests/acceptance/windows-smoke.sh index fe9149f..2ee8e4e 100644 --- a/tests/acceptance/windows-smoke.sh +++ b/tests/acceptance/windows-smoke.sh @@ -54,16 +54,9 @@ powershell -NoProfile -ExecutionPolicy Bypass -File "$HERE/windows-pipe-acl.ps1" || fail "the daemon pipe is not owner-only" echo "--- checkpoint and timeline" -# Wait for the observable baseline, not an assumed startup duration. -baseline_ready=false -for _ in {1..100}; do - if "$ACYCLIC" timeline < /dev/null 2>/dev/null | grep -q baseline; then - baseline_ready=true - break - fi - sleep 0.05 -done -[ "$baseline_ready" = true ] || fail "baseline did not become ready" +# `init` pings the pipeline, whose reply is queued behind its baseline. +"$ACYCLIC" timeline < /dev/null | grep -q baseline \ + || fail "init returned before the baseline was ready" printf 'DIFFERENT AND LONGER CONTENT\n' > src/main.rs printf 'extra\n' > added.txt "$ACYCLIC" checkpoint < /dev/null > /dev/null || fail "checkpoint failed" From dcf3924a4845435ee7d7bde10c4aff0319eab24d Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 14:27:38 +0100 Subject: [PATCH 015/106] Verify Windows daemon restart after promotion --- tests/acceptance/windows-smoke.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/acceptance/windows-smoke.sh b/tests/acceptance/windows-smoke.sh index 2ee8e4e..bc3cb1c 100644 --- a/tests/acceptance/windows-smoke.sh +++ b/tests/acceptance/windows-smoke.sh @@ -91,6 +91,13 @@ printf 'edited in fork\n' > "$fork_dir/a.txt" grep -q 'FORK WORK' note.txt || fail "promote did not land note.txt" grep -q 'edited in fork' a.txt || fail "promote did not land a.txt" +echo "--- daemon restart retains promoted files and timeline" +"$ACYCLIC" stop < /dev/null > /dev/null || fail "stop failed" +"$ACYCLIC" timeline < /dev/null | grep -q baseline \ + || fail "timeline did not recover after restart" +grep -q 'FORK WORK' note.txt || fail "restart lost promoted note.txt" +grep -q 'edited in fork' a.txt || fail "restart lost promoted a.txt" + echo "--- a second daemon refuses the store" if "$ACYCLIC" __daemon "$REPO" < /dev/null > /dev/null 2>&1; then fail "a second daemon started against a live store" From 7460a26208658abfa90d184e5c83a14139bdadc3 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 18:14:16 +0100 Subject: [PATCH 016/106] Lock SDK listing dependencies for plugin lab --- Cargo.lock | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3fdcc3d..953be2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,9 @@ dependencies = [ "bytes", "fs2", "futures", + "getrandom 0.3.4", + "hex", + "imbl", "libc", "prost", "prost-types", @@ -171,6 +174,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "archery" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca55ee147b1926dbea904f50fe4902494e97bc742205abbbf10c709e43815f" + [[package]] name = "arrayvec" version = "0.7.8" @@ -289,6 +298,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "bytes" version = "1.12.1" @@ -728,6 +743,20 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -736,7 +765,7 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", ] [[package]] @@ -936,6 +965,27 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "imbl" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bad832b9b463ed9398b8506488cc2e3b897a9d44f13af115f382aac71f4fec" +dependencies = [ + "archery", + "equivalent", + "imbl-sized-chunks", + "rand_core", + "rand_xoshiro", + "version_check", + "wide", +] + +[[package]] +name = "imbl-sized-chunks" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0813be332553f857953298749fa19549e8b61b80589757c29b4e2a804fa9c6" + [[package]] name = "indexmap" version = "2.14.1" @@ -1418,12 +1468,33 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core", +] + [[package]] name = "ref-cast" version = "1.0.27" @@ -1612,6 +1683,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2188,6 +2268,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -2242,6 +2331,16 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2567,6 +2666,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "zerocopy" version = "0.8.56" From f7f1cff692ea6522f519854a7fc8c41b45adf8f5 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 19:47:26 +0100 Subject: [PATCH 017/106] Centralize plugin replies and harden rewind recovery --- crates/acyclic/Cargo.toml | 1 + crates/acyclic/src/client.rs | 82 +++ crates/acyclic/src/main.rs | 36 +- crates/acyclic/src/mcp.rs | 71 +-- crates/acyclic/src/rewind.rs | 682 ++++++++++++++++++++++--- crates/acyclic/src/server.rs | 9 +- crates/acyclic/tests/restart_rewind.rs | 59 +++ 7 files changed, 787 insertions(+), 153 deletions(-) create mode 100644 crates/acyclic/tests/restart_rewind.rs diff --git a/crates/acyclic/Cargo.toml b/crates/acyclic/Cargo.toml index f28e1cb..3d4d5e9 100644 --- a/crates/acyclic/Cargo.toml +++ b/crates/acyclic/Cargo.toml @@ -40,6 +40,7 @@ windows-sys = { version = "0.61", features = [ "Win32_System_JobObjects", "Win32_System_Memory", "Win32_System_Threading", + "Win32_Storage_FileSystem", ] } [dev-dependencies] diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index e386748..cde39ba 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -142,6 +142,84 @@ impl Client { result } + /// Take a user-requested checkpoint and wait until it has landed. + /// + /// Product consumers should use this instead of constructing the wire + /// operation themselves. Hook-specific checkpoint metadata remains on + /// [`Client::call`] until it has a real consumer-facing abstraction. + pub fn manual_checkpoint( + &mut self, + label: Option, + ) -> Result { + match self.call(proto::Op::Checkpoint { + kind: proto::CheckpointRequestKind::Manual, + session_id: None, + tool_call_id: None, + tool_name: None, + label, + wait: true, + durable: false, + })? { + proto::Reply::Checkpoint(info) => Ok(info), + other => Err(unexpected_reply("checkpoint", &other)), + } + } + + /// List checkpoints through the stable, typed client surface. + pub fn timeline( + &mut self, + session_id: Option, + turn: Option, + limit: u32, + ) -> Result, String> { + match self.call(proto::Op::Timeline { + session_id, + turn, + limit, + })? { + proto::Reply::Timeline(entries) => Ok(entries), + other => Err(unexpected_reply("timeline", &other)), + } + } + + /// Replace the working tree at a checkpoint. + pub fn rewind(&mut self, target: proto::RewindTarget) -> Result { + match self.call(proto::Op::Rewind { target, path: None })? { + proto::Reply::Rewind(info) => Ok(info), + other => Err(unexpected_reply("rewind", &other)), + } + } + + /// Restore one path while leaving the rest of the tree untouched. + pub fn restore(&mut self, checkpoint: i64, path: String) -> Result { + match self.call(proto::Op::Rewind { + target: proto::RewindTarget::Checkpoint(checkpoint), + path: Some(path), + })? { + proto::Reply::Restore(info) => Ok(info), + other => Err(unexpected_reply("restore", &other)), + } + } + + /// Compute a diff using either row ids or generation prefixes. + pub fn diff( + &mut self, + before: Option, + after: Option, + before_hex: Option, + after_hex: Option, + ) -> Result, String> { + match self.call(proto::Op::Diff { + before, + after, + before_hex, + after_hex, + })? { + proto::Reply::Diff(entries) => Ok(entries), + other => Err(unexpected_reply("diff", &other)), + } + } + fn call_inner(&mut self, id: u64, op: proto::Op) -> Result { let deadline = self.call_timeout.map(|timeout| Instant::now() + timeout); let request = proto::Request { @@ -206,6 +284,10 @@ impl Client { } } +fn unexpected_reply(operation: &str, reply: &proto::Reply) -> String { + format!("{operation}: unexpected daemon reply {reply:?}") +} + fn remaining(deadline: Option) -> Result, String> { match deadline { Some(deadline) => { diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index ec23771..244ba75 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -257,6 +257,10 @@ fn main() { // unless a bounded probe shows the mount is genuinely wedged), so the // real tree is back before we read anything. acyclic::fork::reap_dead_shadow(&repo_arg); + let repo_arg = acyclic::rewind::recover_before_repo_open(&repo_arg).unwrap_or_else(|error| { + eprintln!("{}: rewind recovery: {error}", product::NAME); + std::process::exit(1); + }); let repo = repo_arg.canonicalize().unwrap_or_else(|error| { eprintln!("{}: bad repo path: {error}", product::NAME); std::process::exit(1); @@ -625,14 +629,7 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { turn, limit, } => { - let reply = client.call(proto::Op::Timeline { - session_id: session, - turn, - limit, - })?; - let proto::Reply::Timeline(entries) = reply else { - return Err("unexpected reply".into()); - }; + let entries = client.timeline(session, turn, limit)?; if entries.is_empty() { println!("no checkpoints yet"); return Ok(()); @@ -818,13 +815,7 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { } Command::Restore { checkpoint, paths } => { for path in paths { - let reply = client.call(proto::Op::Rewind { - target: proto::RewindTarget::Checkpoint(checkpoint), - path: Some(path), - })?; - let proto::Reply::Restore(info) = reply else { - return Err("unexpected reply".into()); - }; + let info = client.restore(checkpoint, path)?; match info.action { proto::RestoreAction::Removed => { println!("{}: absent at #{}, removed", info.path, info.checkpoint); @@ -863,10 +854,7 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { } } step_aside(); - let reply = client.call(proto::Op::Rewind { target, path: None })?; - let proto::Reply::Rewind(info) = reply else { - return Err("unexpected reply".into()); - }; + let info = client.rewind(target)?; println!("restored checkpoint #{}", info.restored_checkpoint); println!("old tree kept at {}", info.old_tree); println!("note: {}", info.warning); @@ -887,15 +875,7 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { let (after, after_hex) = checkpoint_ref(after.as_deref())?; (before, after, before_hex, after_hex) }; - let reply = client.call(proto::Op::Diff { - before, - after, - before_hex, - after_hex, - })?; - let proto::Reply::Diff(entries) = reply else { - return Err("unexpected reply".into()); - }; + let entries = client.diff(before, after, before_hex, after_hex)?; print_diff(&entries); Ok(()) } diff --git a/crates/acyclic/src/mcp.rs b/crates/acyclic/src/mcp.rs index c468914..c03cac2 100644 --- a/crates/acyclic/src/mcp.rs +++ b/crates/acyclic/src/mcp.rs @@ -104,16 +104,7 @@ where /// recorded afterwards (the same thing the CLI prints), so the caller can /// refer to the post-restore state without another `timeline` call. fn restore_one(client: &mut Client, checkpoint: i64, path: String) -> Result { - let reply = call( - client, - proto::Op::Rewind { - target: proto::RewindTarget::Checkpoint(checkpoint), - path: Some(path), - }, - )?; - let proto::Reply::Restore(info) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let info = client.restore(checkpoint, path).map_err(internal_error)?; let what = match info.action { proto::RestoreAction::Removed => format!("absent at #{}, removed", info.checkpoint), _ => format!("restored from #{}", info.checkpoint), @@ -211,21 +202,9 @@ impl McpServer { Parameters(params): Parameters, ) -> Result { with_daemon(self.repo.clone(), move |client| { - let reply = call( - client, - proto::Op::Checkpoint { - kind: proto::CheckpointRequestKind::Manual, - session_id: None, - tool_call_id: None, - tool_name: None, - label: params.message, - wait: true, - durable: false, - }, - )?; - let proto::Reply::Checkpoint(info) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let info = client + .manual_checkpoint(params.message) + .map_err(internal_error)?; Ok(format!("checkpoint #{} ({})", info.row_id, info.generation)) }) .await @@ -240,17 +219,9 @@ impl McpServer { Parameters(params): Parameters, ) -> Result { with_daemon(self.repo.clone(), move |client| { - let reply = call( - client, - proto::Op::Timeline { - session_id: None, - turn: None, - limit: params.limit.unwrap_or(50), - }, - )?; - let proto::Reply::Timeline(entries) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let entries = client + .timeline(None, None, params.limit.unwrap_or(50)) + .map_err(internal_error)?; if entries.is_empty() { return Ok("no checkpoints yet".into()); } @@ -341,16 +312,9 @@ impl McpServer { )); } with_daemon(self.repo.clone(), move |client| { - let reply = call( - client, - proto::Op::Rewind { - target: proto::RewindTarget::Checkpoint(params.checkpoint), - path: None, - }, - )?; - let proto::Reply::Rewind(info) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let info = client + .rewind(proto::RewindTarget::Checkpoint(params.checkpoint)) + .map_err(internal_error)?; Ok(format!( "restored checkpoint #{}\nold tree kept at {}\nnote: {}", info.restored_checkpoint, info.old_tree, info.warning @@ -366,18 +330,9 @@ impl McpServer { )] async fn diff(&self, Parameters(params): Parameters) -> Result { with_daemon(self.repo.clone(), move |client| { - let reply = call( - client, - proto::Op::Diff { - before: params.before, - after: params.after, - before_hex: None, - after_hex: None, - }, - )?; - let proto::Reply::Diff(entries) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let entries = client + .diff(params.before, params.after, None, None) + .map_err(internal_error)?; if entries.is_empty() { return Ok("no changes".into()); } diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index d974b23..2b4c052 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -4,6 +4,7 @@ //! fully-new — never mixed. use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; #[cfg(unix)] use acyclic_fs::kernel::MetadataField; @@ -19,6 +20,7 @@ use crate::{EngineError, Result}; const MAXIMUM_DIRECTORY_ENTRIES: u32 = 1_024; const MAXIMUM_EXTENT_SPANS: u32 = 65_536; const TRANSFER_BYTES: u64 = 8 * 1024 * 1024; +static PARK_SEQUENCE: AtomicU64 = AtomicU64::new(0); /// What a completed rewind reports back. #[derive(Clone, Debug)] @@ -72,6 +74,7 @@ pub async fn restore_path_into( ) -> Result { let components = validate_relative(relative)?; let destination = root.join(relative); + ensure_real_parents(root, relative)?; let parent = destination .parent() .ok_or_else(|| EngineError::Restore("path has no parent".into()))?; @@ -118,7 +121,7 @@ pub async fn restore_path_into( // Stage next to the destination so the final rename never crosses a // filesystem; the parent must exist (it did at the checkpoint, but the // tree may have lost it since). - std::fs::create_dir_all(parent)?; + ensure_real_parents(root, relative)?; let staged = parent.join(format!( ".{name}.{}-restore-{}", crate::product::NAME, @@ -139,6 +142,7 @@ pub async fn restore_path_into( let _ = remove_any(&staged); return Err(error); } + ensure_real_parents(root, relative)?; // Swap in. An existing destination is exchanged atomically and the old // node discarded; an absent one is a plain rename. @@ -318,6 +322,48 @@ pub(crate) fn validate_relative(relative: &Path) -> Result>> { Ok(components) } +/// Create missing ancestors one component at a time, refusing symlinks and +/// Windows reparse points before a path restore touches its destination. +fn ensure_real_parents(root: &Path, relative: &Path) -> Result<()> { + fn check_real_directory(path: &Path) -> Result<()> { + let metadata = std::fs::symlink_metadata(path)?; + #[cfg(windows)] + let reparse = { + use std::os::windows::fs::MetadataExt; + metadata.file_attributes() & 0x400 != 0 // FILE_ATTRIBUTE_REPARSE_POINT + }; + #[cfg(not(windows))] + let reparse = false; + if !metadata.is_dir() || metadata.file_type().is_symlink() || reparse { + return Err(EngineError::Restore(format!( + "restore parent {} is not a real directory", + path.display() + ))); + } + Ok(()) + } + + check_real_directory(root)?; + let mut cursor = root.to_path_buf(); + if let Some(parent) = relative.parent() { + for component in parent.components() { + let Component::Normal(name) = component else { + continue; + }; + cursor.push(name); + match std::fs::symlink_metadata(&cursor) { + Ok(_) => check_real_directory(&cursor)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir(&cursor)?; + check_real_directory(&cursor)?; + } + Err(error) => return Err(error.into()), + } + } + } + Ok(()) +} + pub(crate) fn namespace_path( components: &[Vec], limits: acyclic_fs::model::VolumeLimits, @@ -400,23 +446,27 @@ fn carry(from: &Path, into: &Path, relative: &[PathBuf]) -> Result<()> { Ok(()) } -/// Best-effort inverse of [`carry`]: every listed path present in `from` -/// and absent in `into` goes back. Used by crash recovery, where the only -/// wrong answer is losing a live copy. -fn move_back(from: &Path, into: &Path, relative: &[PathBuf]) { +/// Inverse of [`carry`]: every listed path present in `from` goes back. +/// A conflict or I/O failure keeps the journal and both trees for retry. +fn move_back(from: &Path, into: &Path, relative: &[PathBuf]) -> Result<()> { for path in relative { let source = from.join(path); let destination = into.join(path); - if std::fs::symlink_metadata(&source).is_err() - || std::fs::symlink_metadata(&destination).is_ok() - { + if !path_exists(&source)? { continue; } + if path_exists(&destination)? { + return Err(EngineError::Restore(format!( + "rewind recovery found both copies of carried path {}", + path.display() + ))); + } if let Some(parent) = destination.parent() { - let _ = std::fs::create_dir_all(parent); + std::fs::create_dir_all(parent)?; } - let _ = std::fs::rename(&source, &destination); + std::fs::rename(&source, &destination)?; } + Ok(()) } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -426,11 +476,101 @@ pub enum Phase { /// Excluded paths moving from repo into tmp. Recovery: move back any /// that already moved, then delete tmp. Carrying, - /// Atomic exchange in flight (or two-step: repo moved aside to tmp2). - /// Recovery: if repo missing, move tmp into place; else delete tmp. + /// Root exchange in flight. Recovery makes the repo whole and parks every + /// displaced tree; Windows may also have an intermediate scratch tree. Swapping, } +/// Result of reconciling an interrupted root replacement. +#[derive(Debug)] +pub struct RecoveredSwap { + /// The target was already published when a Windows parking rename failed. + pub published: bool, + /// The displaced tree retained in trash or a sibling location. + pub old_tree: Option, +} + +/// Stored beside the repository so startup can find the journal even while +/// the Windows exchange has temporarily removed the repository name. The +/// store location may come from a config file inside that missing tree. +#[derive(Serialize, Deserialize)] +struct RecoveryLocator { + repo_root: PathBuf, + journal: PathBuf, + trash: PathBuf, + trash_ttl_days: u32, +} + +fn locator_path(repo_root: &Path) -> Result { + let parent = repo_root + .parent() + .ok_or_else(|| EngineError::Restore("rewind repo root has no parent".into()))?; + let name = repo_root + .file_name() + .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))? + .to_string_lossy(); + Ok(parent.join(format!(".{name}.{}-rewind.json", crate::product::NAME))) +} + +/// Recovers before loading the repository's config or canonicalizing its root. +/// Both operations can fail after the first Windows rename has removed it. +pub fn recover_before_repo_open(repo_root: &Path) -> Result { + let canonical = match repo_root.canonicalize() { + Ok(path) => path, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let parent = repo_root + .parent() + .ok_or_else(|| EngineError::Restore("rewind repo root has no parent".into()))?; + let parent = if parent.as_os_str().is_empty() { + Path::new(".") + } else { + parent + }; + parent.canonicalize()?.join( + repo_root + .file_name() + .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))?, + ) + } + Err(error) => return Err(error.into()), + }; + #[cfg(windows)] + let canonical = { + let mut canonical = canonical; + if let (Some(parent), Some(name)) = (canonical.parent(), canonical.file_name()) { + let name = name.to_string_lossy(); + let suffix = format!(".{}-swap", crate::product::NAME); + if let Some(original) = name + .strip_prefix('.') + .and_then(|name| name.strip_suffix(&suffix)) + { + let candidate = parent.join(original); + if locator_path(&candidate)?.exists() { + // Windows holds the current directory open. Leave the old + // tree before recovery renames it back to the repository. + std::env::set_current_dir(parent)?; + canonical = candidate; + } + } + } + canonical + }; + let locator_path = locator_path(&canonical)?; + let text = match std::fs::read_to_string(&locator_path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(canonical), + Err(error) => return Err(error.into()), + }; + let locator: RecoveryLocator = serde_json::from_str(&text) + .map_err(|error| EngineError::Restore(format!("rewind locator: {error}")))?; + if locator.repo_root != canonical { + return Err(EngineError::Restore("rewind locator repo mismatch".into())); + } + recover(&locator.journal, &locator.trash, locator.trash_ttl_days)?; + std::fs::remove_file(locator_path)?; + Ok(canonical) +} + /// Materializes `generation` into `destination`, which must not exist yet. /// Used for copy-mode forks; a rewind does the same into its temp sibling. pub async fn materialize_into( @@ -482,6 +622,11 @@ pub async fn execute( let nonce = std::process::id(); let tmp = parent.join(format!(".{name}.{}-tmp-{nonce}", crate::product::NAME)); let journal_path = store.paths.rewind_journal(); + let trash_root = store.paths.trash(); + + // A failed previous exchange may have left a complete tree in scratch. + // Resolve its journal before reusing either the temporary name or journal. + recover(&journal_path, &trash_root, trash_ttl_days)?; // 1. Materialize the target into an empty sibling directory. A tmp left // by an earlier attempt that failed before the swap is stale by @@ -542,7 +687,7 @@ pub async fn execute( )?; if let Err(error) = carry(repo, &tmp, &carried) { // Undo what moved, then abandon the rewind with the repo whole. - move_back(&tmp, repo, &carried); + move_back(&tmp, repo, &carried)?; let _ = std::fs::remove_dir_all(&tmp); let _ = std::fs::remove_file(&journal_path); return Err(error); @@ -560,14 +705,24 @@ pub async fn execute( carried, }, )?; - atomic_exchange(repo, &tmp)?; + let locator = publish_locator(repo, &journal_path, &trash_root, trash_ttl_days)?; + if let Err(error) = atomic_exchange(repo, &tmp) { + // The third Windows rename can fail after the new tree is already + // published. Reconcile immediately, before the daemon accepts another + // rewind that could reuse the scratch name. + return reconcile_exchange_failure( + target, + repo, + &journal_path, + &trash_root, + trash_ttl_days, + error, + ); + } // 4. Old tree to trash (best effort: EXDEV falls back to a sibling path). - let trash_root = store.paths.trash(); let old_tree = park_replaced_tree(&tmp, &trash_root, parent, &name)?; - std::fs::remove_file(&journal_path) - .map_err(|error| EngineError::Restore(format!("rewind: clear journal: {error}")))?; - prune_trash(&trash_root, trash_ttl_days); + finish_published_rewind(&journal_path, &locator, &trash_root, trash_ttl_days); Ok(RewindOutcome { restored: target, @@ -576,6 +731,41 @@ pub async fn execute( }) } +fn finish_published_rewind(journal: &Path, locator: &Path, trash: &Path, ttl_days: u32) { + // The tree is already published. Cleanup failure leaves the journal and + // locator for startup retry, but must not report a failed rewind. + if std::fs::remove_file(journal).is_ok() { + let _ = std::fs::remove_file(locator); + } + prune_trash(trash, ttl_days); +} + +fn reconcile_exchange_failure( + target: GenerationId, + repo: &Path, + journal_path: &Path, + trash_root: &Path, + trash_ttl_days: u32, + error: EngineError, +) -> Result { + let recovered = recover(journal_path, trash_root, trash_ttl_days)?; + if let Ok(locator) = locator_path(repo) { + let _ = std::fs::remove_file(locator); + } + if let Some(RecoveredSwap { + published: true, + old_tree: Some(old_tree), + }) = recovered + { + return Ok(RewindOutcome { + restored: target, + old_tree, + warning: "reload your editor: open files still point at the replaced tree", + }); + } + Err(error) +} + /// Moves the replaced tree out of the way and returns where it landed. /// /// The trash lives in the store, which can be on another volume than the @@ -584,13 +774,18 @@ pub async fn execute( fn park_replaced_tree(tmp: &Path, trash_root: &Path, parent: &Path, name: &str) -> Result { let stamp = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |duration| duration.as_secs()); - let trashed = trash_root.join(format!("{name}-{stamp}")); - if std::fs::rename(tmp, &trashed).is_ok() { + .map_or(0, |duration| duration.as_nanos()); + let unique = format!( + "{stamp}-{}-{}", + std::process::id(), + PARK_SEQUENCE.fetch_add(1, Ordering::Relaxed) + ); + let trashed = trash_root.join(format!("{name}-{unique}")); + if durable_rename(tmp, &trashed, false).is_ok() { return Ok(trashed); } - let sibling = parent.join(format!(".{name}.{}-trash-{stamp}", crate::product::NAME)); - std::fs::rename(tmp, &sibling).map_err(|error| { + let sibling = parent.join(format!(".{name}.{}-trash-{unique}", crate::product::NAME)); + durable_rename(tmp, &sibling, false).map_err(|error| { EngineError::Restore(format!( "rewind: park the replaced tree at {}: {error}", sibling.display() @@ -601,7 +796,11 @@ fn park_replaced_tree(tmp: &Path, trash_root: &Path, parent: &Path, name: &str) /// Startup crash recovery. Reads the journal (if any) and finishes or unwinds /// the interrupted rewind so the repo is whole before the pipeline baselines. -pub fn recover(journal_path: &Path) -> Result> { +pub fn recover( + journal_path: &Path, + trash_root: &Path, + trash_ttl_days: u32, +) -> Result> { let text = match std::fs::read_to_string(journal_path) { Ok(text) => text, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), @@ -609,6 +808,11 @@ pub fn recover(journal_path: &Path) -> Result> { }; let journal: Journal = serde_json::from_str(&text) .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; + #[cfg(windows)] + let mut published = false; + #[cfg(not(windows))] + let published = false; + let mut old_tree = None; match journal.phase { Phase::Materializing => { // Repo untouched; the partial tmp tree is garbage. @@ -617,39 +821,96 @@ pub fn recover(journal_path: &Path) -> Result> { Phase::Carrying => { // Some excluded paths may already sit in tmp: bring them home, // then drop the unused new tree. - move_back(&journal.tmp, &journal.repo_root, &journal.carried); + move_back(&journal.tmp, &journal.repo_root, &journal.carried)?; let _ = std::fs::remove_dir_all(&journal.tmp); } Phase::Swapping => { - // Windows swaps through a scratch name (see `atomic_exchange`); - // whichever step it died on, the scratch holds a tree that one - // of the branches below is about to supersede. #[cfg(windows)] let scratch = swap_scratch(&journal.repo_root); - if !journal.repo_root.exists() && journal.tmp.exists() { - // The swap died with the repo path vacated and the new tree - // still parked at tmp: finish the move. The carried paths are - // inside tmp and come along. - std::fs::rename(&journal.tmp, &journal.repo_root)?; + #[cfg(windows)] + let scratch_present = scratch + .as_deref() + .map(path_exists) + .transpose()? + .unwrap_or(false); + let repo_present = path_exists(&journal.repo_root)?; + let tmp_present = path_exists(&journal.tmp)?; + #[cfg(windows)] + { + published = repo_present && scratch_present && !tmp_present; + } + #[cfg(windows)] + if scratch_present && tmp_present && repo_present { + return Err(EngineError::Restore( + "rewind recovery found three live trees; refusing to discard any".into(), + )); + } + #[cfg(windows)] + if !repo_present && scratch_present { + // Rename #1 landed but the publication may not have. Restore + // the old tree, including excluded paths carried into tmp. + let old = scratch + .as_deref() + .ok_or_else(|| EngineError::Restore("rewind scratch path is missing".into()))?; + move_back(&journal.tmp, old, &journal.carried)?; + durable_rename(old, &journal.repo_root, false)?; + } else if !repo_present && tmp_present { + // A journal from an older implementation can have no scratch. + // Make the repo whole before attempting store startup. + durable_rename(&journal.tmp, &journal.repo_root, false)?; } else { - // The repo path is whole, so it names exactly one tree and - // tmp holds the other: the old tree (swap done — keep it out - // of the way) or the unused new tree (swap never happened). - // Carried paths live in whichever tree is new: if that is - // still tmp, they must come back before tmp goes. - move_back(&journal.tmp, &journal.repo_root, &journal.carried); - let _ = std::fs::remove_dir_all(&journal.tmp); + // If the exchange never started, carried paths are still in + // tmp and must return to the live tree. After a completed + // exchange they are already in the live tree. + move_back(&journal.tmp, &journal.repo_root, &journal.carried)?; + } + #[cfg(not(windows))] + if !repo_present && tmp_present { + durable_rename(&journal.tmp, &journal.repo_root, false)?; + } else { + move_back(&journal.tmp, &journal.repo_root, &journal.carried)?; + } + if !path_exists(&journal.repo_root)? { + return Err(EngineError::Restore( + "rewind recovery could not find a complete repository tree".into(), + )); + } + let parent = journal + .repo_root + .parent() + .ok_or_else(|| EngineError::Restore("rewind repo root has no parent".into()))?; + let name = journal + .repo_root + .file_name() + .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))? + .to_string_lossy(); + if path_exists(&journal.tmp)? { + old_tree = Some(park_replaced_tree(&journal.tmp, trash_root, parent, &name)?); } - // Whatever the scratch still holds has now been superseded by the - // branch above, exactly as `tmp` is discarded there. #[cfg(windows)] if let Some(scratch) = scratch { - let _ = remove_any(&scratch); + if path_exists(&scratch)? { + old_tree = Some(park_replaced_tree(&scratch, trash_root, parent, &name)?); + } } } } std::fs::remove_file(journal_path)?; - Ok(Some(journal)) + #[cfg(unix)] + sync_parent(journal_path)?; + prune_trash(trash_root, trash_ttl_days); + Ok(Some(RecoveredSwap { + published, + old_tree, + })) +} + +fn path_exists(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } } fn write_journal(path: &Path, journal: &Journal) -> Result<()> { @@ -670,7 +931,7 @@ fn write_journal(path: &Path, journal: &Journal) -> Result<()> { file.write_all(text.as_bytes())?; file.sync_all()?; drop(file); - std::fs::rename(&tmp, path) + durable_rename(&tmp, path, true) }; write().map_err(|error| { let _ = std::fs::remove_file(&tmp); @@ -679,6 +940,97 @@ fn write_journal(path: &Path, journal: &Journal) -> Result<()> { Ok(()) } +fn write_locator(path: &Path, locator: &RecoveryLocator) -> Result<()> { + use std::io::Write; + let text = serde_json::to_vec(locator) + .map_err(|error| EngineError::Restore(format!("encode rewind locator: {error}")))?; + let tmp = path.with_extension("tmp"); + let mut file = std::fs::File::create(&tmp)?; + file.write_all(&text)?; + file.sync_all()?; + drop(file); + durable_rename(&tmp, path, true)?; + Ok(()) +} + +fn publish_locator( + repo: &Path, + journal: &Path, + trash: &Path, + trash_ttl_days: u32, +) -> Result { + let path = locator_path(repo)?; + write_locator( + &path, + &RecoveryLocator { + repo_root: repo.to_path_buf(), + journal: journal.to_path_buf(), + trash: trash.to_path_buf(), + trash_ttl_days, + }, + )?; + Ok(path) +} + +#[cfg(unix)] +fn durable_rename(from: &Path, to: &Path, _replace: bool) -> std::io::Result<()> { + std::fs::rename(from, to)?; + let parent = to + .parent() + .ok_or_else(|| std::io::Error::other("rename path has no parent"))?; + std::fs::File::open(parent)?.sync_all()?; + if from.parent() != to.parent() { + let parent = from + .parent() + .ok_or_else(|| std::io::Error::other("rename path has no parent"))?; + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +#[cfg(windows)] +#[allow( + unsafe_code, + reason = "MoveFileExW receives two terminated UTF-16 path buffers" +)] +fn durable_rename(from: &Path, to: &Path, replace: bool) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::{ + MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, + }; + let from: Vec = from + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let to: Vec = to + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let flags = MOVEFILE_WRITE_THROUGH + | if replace { + MOVEFILE_REPLACE_EXISTING + } else { + 0 + }; + if unsafe { MoveFileExW(from.as_ptr(), to.as_ptr(), flags) } == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(unix)] +fn sync_parent(path: &Path) -> Result<()> { + std::fs::File::open( + path.parent() + .ok_or_else(|| EngineError::Restore("rewind path has no parent".into()))?, + )? + .sync_all()?; + Ok(()) +} + fn prune_trash(trash_root: &Path, ttl_days: u32) { let Ok(entries) = std::fs::read_dir(trash_root) else { return; @@ -721,28 +1073,33 @@ pub(crate) fn swap_scratch(a: &Path) -> Option { /// path either absent or naming exactly one whole tree, and [`recover`] /// resolves each of them from the journal: the `Swapping` arm finishes the /// move when the repo path is missing, and clears the scratch either way. -/// A crash during a journal-less [`restore_path`] leaves the same scratch -/// behind, which the next swap of that path removes before it starts. +/// A crash during a journal-less [`restore_path`] may leave the same scratch +/// behind; a later swap refuses to overwrite that potentially unique tree. #[cfg(windows)] fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { let scratch = swap_scratch(a).ok_or_else(|| EngineError::Restore("swap path has no parent".into()))?; - // A scratch left by an interrupted swap is stale by construction: the - // recovery below never keeps it, so anything still here predates us. - let _ = remove_any(&scratch); + // A preexisting scratch may be the only copy of the prior tree after a + // failed exchange. The caller must recover it before starting a new swap. + if path_exists(&scratch)? { + return Err(EngineError::Restore(format!( + "swap scratch {} is occupied; recover the previous exchange first", + scratch.display() + ))); + } - std::fs::rename(a, &scratch).map_err(|error| { + durable_rename(a, &scratch, false).map_err(|error| { EngineError::Restore(format!("swap: move aside {}: {error}", a.display())) })?; - if let Err(error) = std::fs::rename(b, a) { + if let Err(error) = durable_rename(b, a, false) { // Nothing has been published yet; put `a` back and fail clean. - let _ = std::fs::rename(&scratch, a); + let _ = durable_rename(&scratch, a, false); return Err(EngineError::Restore(format!( "swap: move {} into place: {error}", b.display() ))); } - std::fs::rename(&scratch, b).map_err(|error| { + durable_rename(&scratch, b, false).map_err(|error| { // `a` already holds the new tree, so the exchange has effectively // happened; only the old tree's parking spot is wrong. Leave the // scratch for recovery rather than unwinding a published swap. @@ -766,6 +1123,10 @@ fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { // atomically on APFS. let result = unsafe { libc::renamex_np(a_c.as_ptr(), b_c.as_ptr(), libc::RENAME_SWAP) }; if result == 0 { + sync_parent(a)?; + if a.parent() != b.parent() { + sync_parent(b)?; + } Ok(()) } else { Err(EngineError::Restore(format!( @@ -799,6 +1160,10 @@ fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { ) }; if result == 0 { + sync_parent(a)?; + if a.parent() != b.parent() { + sync_parent(b)?; + } Ok(()) } else { Err(EngineError::Restore(format!( @@ -811,6 +1176,75 @@ fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { mod tests { use super::*; + #[test] + fn parking_replaced_trees_never_reuses_a_name() -> Result<()> { + let work = tempfile::tempdir()?; + let trash = work.path().join("trash"); + std::fs::create_dir(&trash)?; + let mut parked = Vec::new(); + for index in 0..3 { + let tmp = work.path().join(format!("tmp-{index}")); + std::fs::create_dir(&tmp)?; + std::fs::write(tmp.join("old.txt"), index.to_string())?; + let destination = park_replaced_tree(&tmp, &trash, work.path(), "repo")?; + assert_eq!( + std::fs::read_to_string(destination.join("old.txt"))?, + index.to_string() + ); + parked.push(destination); + } + assert!(parked[0] != parked[1] && parked[1] != parked[2] && parked[0] != parked[2]); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn path_restore_rejects_symlinked_parent() -> Result<()> { + let work = tempfile::tempdir()?; + let root = work.path().join("repo"); + let outside = work.path().join("outside"); + std::fs::create_dir(&root)?; + std::fs::create_dir(&outside)?; + std::os::unix::fs::symlink(&outside, root.join("dir"))?; + assert!(ensure_real_parents(&root, Path::new("dir/file")).is_err()); + assert!(!outside.join("file").exists()); + Ok(()) + } + + #[cfg(windows)] + #[test] + fn path_restore_rejects_reparse_parent_when_symlinks_are_available() -> Result<()> { + let work = tempfile::tempdir()?; + let root = work.path().join("repo"); + let outside = work.path().join("outside"); + std::fs::create_dir(&root)?; + std::fs::create_dir(&outside)?; + if let Err(error) = std::os::windows::fs::symlink_dir(&outside, root.join("dir")) { + if error.kind() == std::io::ErrorKind::PermissionDenied { + return Ok(()); + } + return Err(error.into()); + } + assert!(ensure_real_parents(&root, Path::new("dir/file")).is_err()); + assert!(!outside.join("file").exists()); + Ok(()) + } + + fn recover(journal_path: &Path) -> Result> { + let trash = journal_path.with_file_name("trash"); + std::fs::create_dir_all(&trash)?; + super::recover(journal_path, &trash, 30) + } + + fn parked_tree_contains(journal_path: &Path, file: &str, expected: &[u8]) -> bool { + let trash = journal_path.with_file_name("trash"); + std::fs::read_dir(trash) + .into_iter() + .flatten() + .filter_map(std::result::Result::ok) + .any(|entry| std::fs::read(entry.path().join(file)).ok().as_deref() == Some(expected)) + } + fn journal(repo: &Path, tmp: &Path, phase: Phase) -> Journal { Journal { target_generation: "00".repeat(32), @@ -835,7 +1269,10 @@ mod tests { let journal_path = work.path().join("journal.json"); write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); - recover(&journal_path).expect("recover"); + let recovered = recover(&journal_path) + .expect("recover") + .expect("recovered journal"); + assert!(!recovered.published); assert_eq!( std::fs::read(repo.join("file.txt")).expect("read"), b"restored" @@ -921,6 +1358,32 @@ mod tests { assert!(!tmp.exists()); } + #[test] + fn recover_keeps_both_trees_and_journal_on_carried_path_conflict() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir(&repo).expect("repo"); + std::fs::create_dir(&tmp).expect("tmp"); + std::fs::write(repo.join(".env"), b"repo copy").expect("repo data"); + std::fs::write(tmp.join(".env"), b"staged copy").expect("staged data"); + let journal_path = work.path().join("journal.json"); + let mut entry = journal(&repo, &tmp, Phase::Carrying); + entry.carried = vec![PathBuf::from(".env")]; + write(&journal_path, &entry); + + recover(&journal_path).expect_err("conflicting copies require inspection"); + assert_eq!( + std::fs::read(repo.join(".env")).expect("repo"), + b"repo copy" + ); + assert_eq!( + std::fs::read(tmp.join(".env")).expect("staged"), + b"staged copy" + ); + assert!(journal_path.exists()); + } + #[test] fn recover_after_the_swap_leaves_carried_paths_in_the_new_tree() { // Exchange completed: repo is the new tree with the carried paths, @@ -945,6 +1408,7 @@ mod tests { b"LIVE" ); assert!(!tmp.exists()); + assert!(parked_tree_contains(&journal_path, "file.txt", b"old")); } #[test] @@ -1004,41 +1468,110 @@ mod tests { ); } - /// Windows swaps through a scratch directory, so a crash can leave the - /// repo path vacated with the new tree still at `tmp` and the old tree - /// parked in the scratch. Recovery has to finish the move *and* clear the - /// scratch, or the next rewind inherits a stale tree beside the repo. #[cfg(windows)] #[test] - fn recover_clears_the_scratch_a_windows_swap_left() { + fn exchange_refuses_to_overwrite_a_previous_scratch_tree() { + let work = tempfile::tempdir().expect("tempdir"); + let live = work.path().join("live"); + let staged = work.path().join("staged"); + let scratch = swap_scratch(&live).expect("scratch"); + for path in [&live, &staged, &scratch] { + std::fs::create_dir(path).expect("tree"); + } + std::fs::write(scratch.join("only-copy"), b"old data").expect("scratch data"); + + atomic_exchange(&live, &staged).expect_err("scratch must block new swap"); + assert!(live.exists()); + assert!(staged.exists()); + assert_eq!( + std::fs::read(scratch.join("only-copy")).expect("old data survived"), + b"old data" + ); + } + + /// A crash after Windows rename #1 must roll back to the old tree and + /// retain the staged new tree in trash for recovery or inspection. + #[cfg(windows)] + #[test] + fn startup_recovers_before_the_repository_path_exists() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work + .path() + .canonicalize() + .expect("canonical parent") + .join("repo"); + let scratch = swap_scratch(&repo).expect("scratch path"); + let staged = work.path().join("staged"); + std::fs::create_dir(&scratch).expect("old tree"); + std::fs::write(scratch.join("old.txt"), b"original").expect("old file"); + std::fs::create_dir(&staged).expect("new tree"); + std::fs::write(staged.join("new.txt"), b"replacement").expect("new file"); + let store = work.path().join("custom-store"); + std::fs::create_dir(&store).expect("custom store"); + let journal_path = store.join("rewind-journal.json"); + let trash = store.join("trash"); + std::fs::create_dir(&trash).expect("trash"); + write(&journal_path, &journal(&repo, &staged, Phase::Swapping)); + let locator = locator_path(&repo).expect("locator path"); + write_locator( + &locator, + &RecoveryLocator { + repo_root: repo.clone(), + journal: journal_path.clone(), + trash, + trash_ttl_days: 30, + }, + ) + .expect("locator"); + + recover_before_repo_open(&repo).expect("startup recovery"); + assert_eq!( + std::fs::read(repo.join("old.txt")).expect("old tree"), + b"original" + ); + assert!(!journal_path.exists()); + assert!(!locator.exists()); + } + + #[cfg(windows)] + #[test] + fn recover_preserves_both_trees_after_first_windows_rename() { let work = tempfile::tempdir().expect("tempdir"); let repo = work.path().join("repo"); let tmp = work.path().join("repo.tmp"); std::fs::create_dir(&tmp).expect("tmp"); std::fs::write(tmp.join("file.txt"), b"new tree").expect("seed new"); + std::fs::write(tmp.join(".env"), b"carried live data").expect("carry"); // Died between rename one and rename two: repo vacated, old tree parked. let scratch = swap_scratch(&repo).expect("scratch path"); std::fs::create_dir(&scratch).expect("scratch"); std::fs::write(scratch.join("file.txt"), b"old tree").expect("seed old"); let journal_path = work.path().join("journal.json"); - write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); + let mut entry = journal(&repo, &tmp, Phase::Swapping); + entry.carried = vec![PathBuf::from(".env")]; + write(&journal_path, &entry); recover(&journal_path).expect("recover"); assert_eq!( std::fs::read(repo.join("file.txt")).expect("repo whole"), - b"new tree" + b"old tree" ); assert!(!scratch.exists(), "scratch must not outlive recovery"); assert!(!tmp.exists()); assert!(!journal_path.exists()); + assert_eq!( + std::fs::read(repo.join(".env")).expect("carried data survived"), + b"carried live data" + ); + assert!(parked_tree_contains(&journal_path, "file.txt", b"new tree")); } - /// The other Windows crash point: rename two landed, so the repo already - /// holds the new tree and only the scratch is left to clear. + /// After rename #2 the new tree is published, and the old tree in + /// scratch must be parked rather than deleted. #[cfg(windows)] #[test] - fn recover_clears_the_scratch_after_the_swap_landed() { + fn recover_preserves_old_tree_after_second_windows_rename() { let work = tempfile::tempdir().expect("tempdir"); let repo = work.path().join("repo"); std::fs::create_dir(&repo).expect("repo"); @@ -1050,7 +1583,23 @@ mod tests { let journal_path = work.path().join("journal.json"); write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); - recover(&journal_path).expect("recover"); + let generation = GenerationId::new(acyclic_fs::Digest::ZERO); + let trash = journal_path.with_file_name("trash"); + std::fs::create_dir(&trash).expect("trash"); + let outcome = reconcile_exchange_failure( + generation, + &repo, + &journal_path, + &trash, + 30, + EngineError::Restore("injected third rename failure".into()), + ) + .expect("published rewind is a success"); + assert_eq!(outcome.restored, generation); + assert_eq!( + std::fs::read(outcome.old_tree.join("file.txt")).expect("old tree"), + b"old tree" + ); assert_eq!( std::fs::read(repo.join("file.txt")).expect("repo whole"), @@ -1058,5 +1607,6 @@ mod tests { ); assert!(!scratch.exists(), "scratch must not outlive recovery"); assert!(!journal_path.exists()); + assert!(parked_tree_contains(&journal_path, "file.txt", b"old tree")); } } diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index b64cd4e..4584b41 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -185,6 +185,8 @@ pub fn run(repo_root: &Path) -> Result<(), String> { ); phase = std::time::Instant::now(); }; + rewind::recover_before_repo_open(repo_root).map_err(|error| error.to_string())?; + lap("rewind recovery before repo open"); fork::sweep_stale_dry_session(repo_root); lap("sweep stale dry-run session"); @@ -197,7 +199,12 @@ pub fn run(repo_root: &Path) -> Result<(), String> { // Finish or unwind any rewind that a crash interrupted BEFORE the store // opens and the pipeline baselines; sweep fork dirs a dead daemon left // mounted (fork sessions do not survive the daemon). - rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?; + rewind::recover( + &paths.rewind_journal(), + &paths.trash(), + config.trash_ttl_days, + ) + .map_err(|error| error.to_string())?; lap("rewind journal recovery"); fork::sweep_stale_forks(repo_root); lap("sweep stale forks"); diff --git a/crates/acyclic/tests/restart_rewind.rs b/crates/acyclic/tests/restart_rewind.rs new file mode 100644 index 0000000..a9c7683 --- /dev/null +++ b/crates/acyclic/tests/restart_rewind.rs @@ -0,0 +1,59 @@ +//! Recovery has to run through the real CLI entry point before it can +//! canonicalize a repository name temporarily absent during Windows rewind. + +#[cfg(windows)] +#[test] +fn cli_recovers_a_missing_repo_before_loading_config() -> Result<(), Box> { + for default_repo in [false, true] { + let work = tempfile::tempdir()?; + let parent = work.path().canonicalize()?; + let repo = parent.join("repo"); + let scratch = parent.join(format!(".repo.{}-swap", acyclic::product::NAME)); + let staged = parent.join("staged"); + std::fs::create_dir(&scratch)?; + std::fs::write(scratch.join("old.txt"), b"original")?; + std::fs::create_dir(&staged)?; + let store = parent.join("custom-store"); + let trash = store.join("trash"); + std::fs::create_dir_all(&trash)?; + let journal = store.join("rewind-journal.json"); + std::fs::write( + &journal, + serde_json::json!({ + "target_generation": "00", + "repo_root": repo, + "tmp": staged, + "phase": "Swapping", + "carried": [], + }) + .to_string(), + )?; + let locator = parent.join(format!(".repo.{}-rewind.json", acyclic::product::NAME)); + std::fs::write( + &locator, + serde_json::json!({ + "repo_root": repo, + "journal": journal, + "trash": trash, + "trash_ttl_days": 30, + }) + .to_string(), + )?; + + let mut command = std::process::Command::new(env!("CARGO_BIN_EXE_acyclic")); + command.current_dir(if default_repo { &scratch } else { &parent }); + if !default_repo { + command.arg("--repo").arg("repo"); + } + let output = command.arg("policy").output()?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(std::fs::read(repo.join("old.txt"))?, b"original"); + assert!(!journal.exists()); + assert!(!locator.exists()); + } + Ok(()) +} From fc7f5e280b9678fbe2c8c9b034d6a23790b1042c Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 20:31:55 +0100 Subject: [PATCH 018/106] Use SDK materialization for restore and recover hard-link rescans --- crates/acyclic/src/merge.rs | 27 +-- crates/acyclic/src/pipeline.rs | 221 +++++++++++++-------- crates/acyclic/src/rewind.rs | 326 ++++++++++++------------------- crates/acyclic/tests/pipeline.rs | 28 +++ 4 files changed, 301 insertions(+), 301 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 029b51b..99b3922 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -1134,31 +1134,12 @@ pub async fn materialize_paths( paths: &[PathBuf], ) -> Result<()> { let mut checkout = store.checkout_exact(generation).await?; - let limits = checkout.volume_config().limits; - let cancel = CancellationToken::new(); for path in paths { - let namespace = namespace_of(path)?; - let destination = dir.join(path); - crate::rewind::remove_any(&destination)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - let Some(record) = lookup.record else { - continue; - }; - if let Some(parent) = destination.parent() { - std::fs::create_dir_all(parent)?; - } - crate::rewind::write_node( + crate::rewind::materialize_path_into_checkout( &mut checkout, - &namespace, - record.kind, - &record.payload, - &destination, - limits, - &cancel, + dir, + path, + crate::rewind::PathReplace::LiveMount, ) .await?; } diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 8e86d9e..809979b 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -932,6 +932,10 @@ impl Pipeline { /// Full baseline: capture the whole tree, checkpoint, finish the watcher /// rescan, publish. Used at startup and after RescanRequired/rewind. + #[allow( + clippy::too_many_lines, + reason = "rescan retries share one publication boundary" + )] async fn baseline(&mut self, kind: CheckpointKind) -> Result<()> { crate::trace!( "pipeline", @@ -939,83 +943,118 @@ impl Pipeline { ); let baseline_started = Instant::now(); self.state = State::Baselining; - // A baseline requires a clean checkout. Mid-session (watcher - // invalidation, rewind) the overlay holds uncommitted captures: - // publish them first. At startup this is a no-op. - let phase = Instant::now(); - self.commit_engine().await?; - let precommit_ms = crate::trace::ms(phase); - let phase = Instant::now(); - capture_baseline( - &mut self.store.checkout, - &self.options, - WorkCounters::UNBOUNDED, - &self.cancel, - ) - .await - .map_err(EngineError::fs("capture baseline"))?; - let capture_ms = crate::trace::ms(phase); - let phase = Instant::now(); - self.scrub_exclusions().await?; - let generation = self.checkpoint_engine().await?; - let snapshot_ms = crate::trace::ms(phase); - let row = self - .index - .record(generation, kind, &Attribution::default())?; - self.last_generation = generation; - self.last_checkpoint_row = Some(row); - - // Changes that raced the baseline arrive as the rescan-completion - // batch; fold them in before declaring Ready. - let batch = self - .watch - .as_mut() - .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? - .finish_rescan() - .map_err(EngineError::fs("finish rescan"))?; - // A root hint here is already covered by the rescan that just ran. - let (batch, root) = strip_root_hints(batch); - if root != RootHint::None { + // A newly created hard link can race the rescan tail. Restart with a + // fresh watcher so the next full capture sees every alias together. + for attempt in 0..3 { + // A baseline requires a clean checkout. Mid-session (watcher + // invalidation, rewind) the overlay holds uncommitted captures: + // publish them first. At startup this is a no-op. + let phase = Instant::now(); + self.commit_engine().await?; + let precommit_ms = crate::trace::ms(phase); + let phase = Instant::now(); + capture_baseline( + &mut self.store.checkout, + &self.options, + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await + .map_err(EngineError::fs("capture baseline"))?; + let capture_ms = crate::trace::ms(phase); + let phase = Instant::now(); + self.scrub_exclusions().await?; + let scrub_ms = crate::trace::ms(phase); + // Changes that raced the baseline arrive as the rescan-completion + // batch; fold them in before declaring Ready. + let batch = self + .watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? + .finish_rescan() + .map_err(EngineError::fs("finish rescan"))?; + // A root hint here is already covered by the rescan that just ran. + let (batch, root) = strip_root_hints(batch); + if matches!(batch, WatchBatch::RescanRequired { .. }) { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some("rescan tail invalidated".into()); + self.reset_watch().await?; + crate::trace!( + "pipeline", + "baseline rescan tail invalidated; retry {}", + attempt + 1 + ); + continue; + } + if root != RootHint::None { + crate::trace!( + "pipeline", + "rescan tail: root hint ({root:?}) dropped, covered by the rescan" + ); + } + if let WatchBatch::Changes { ref changes, .. } = batch { + if !changes.is_empty() { + let captured = capture_watch_batch( + &mut self.store.checkout, + batch, + &self.options, + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await; + if let Err(failure) = captured { + if matches!( + failure.error, + acyclic_fs::CaptureError::RescanRequired { .. } + ) { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some(failure.error.to_string()); + self.reset_watch().await?; + crate::trace!( + "pipeline", + "baseline rescan tail needs full capture; retry {}", + attempt + 1 + ); + continue; + } + return Err(EngineError::fs("capture rescan tail")(failure)); + } + self.scrub_exclusions().await?; + } + } + let phase = Instant::now(); + let generation = self.checkpoint_engine().await?; + let snapshot_ms = crate::trace::ms(phase); + let row = self + .index + .record(generation, kind, &Attribution::default())?; + self.last_generation = generation; + self.last_checkpoint_row = Some(row); + let phase = Instant::now(); + self.commit_engine().await?; + let postcommit_ms = crate::trace::ms(phase); + self.state = State::Ready; + if kind == CheckpointKind::Recovered { + let ms = crate::trace::ms(baseline_started); + self.watcher_health.recovery_rescans += 1; + self.watcher_health.recovery_ms_total += ms; + self.watcher_health.last_recovery_ms = ms; + } crate::trace!( "pipeline", - "rescan tail: root hint ({root:?}) dropped, covered by the rescan" + "baseline phases: pre-commit {precommit_ms:.1}ms, full capture {capture_ms:.1}ms, \ + scrub {scrub_ms:.1}ms, snapshot {snapshot_ms:.1}ms, post-commit {postcommit_ms:.1}ms" ); + crate::trace!( + "pipeline", + "baseline done in {:.1}ms; state Ready", + crate::trace::ms(baseline_started) + ); + return Ok(()); } - if let WatchBatch::Changes { ref changes, .. } = batch { - if !changes.is_empty() { - capture_watch_batch( - &mut self.store.checkout, - batch, - &self.options, - WorkCounters::UNBOUNDED, - &self.cancel, - ) - .await - .map_err(EngineError::fs("capture rescan tail"))?; - self.scrub_exclusions().await?; - } - } - let phase = Instant::now(); - self.commit_engine().await?; - let postcommit_ms = crate::trace::ms(phase); - self.state = State::Ready; - if kind == CheckpointKind::Recovered { - let ms = crate::trace::ms(baseline_started); - self.watcher_health.recovery_rescans += 1; - self.watcher_health.recovery_ms_total += ms; - self.watcher_health.last_recovery_ms = ms; - } - crate::trace!( - "pipeline", - "baseline phases: pre-commit {precommit_ms:.1}ms, full capture {capture_ms:.1}ms, \ - scrub+snapshot {snapshot_ms:.1}ms, rescan tail+post-commit {postcommit_ms:.1}ms" - ); - crate::trace!( - "pipeline", - "baseline done in {:.1}ms; state Ready", - crate::trace::ms(baseline_started) - ); - Ok(()) + Err(EngineError::Store( + "baseline rescan repeatedly invalidated".into(), + )) } /// Handles one request; returns true when the pipeline should exit. @@ -1446,15 +1485,27 @@ impl Pipeline { WatchBatch::Changes { ref changes, .. } if !changes.is_empty() => { batches += 1; hints += changes.len(); - capture_watch_batch( + let captured = capture_watch_batch( &mut self.store.checkout, batch, &self.options, WorkCounters::UNBOUNDED, &self.cancel, ) - .await - .map_err(EngineError::fs("capture watch batch"))?; + .await; + if let Err(failure) = captured { + if matches!( + failure.error, + acyclic_fs::CaptureError::RescanRequired { .. } + ) { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some(failure.error.to_string()); + self.reset_watch().await?; + self.baseline(CheckpointKind::Recovered).await?; + return Ok(true); + } + return Err(EngineError::fs("capture watch batch")(failure)); + } if scrub { self.scrub_exclusions().await?; } @@ -1739,7 +1790,7 @@ impl Pipeline { &parent, )?)); } - capture_watch_batch( + let captured = capture_watch_batch( &mut self.store.checkout, WatchBatch::Changes { epoch: WatchEpoch::from_u64(0), @@ -1751,8 +1802,20 @@ impl Pipeline { WorkCounters::UNBOUNDED, &self.cancel, ) - .await - .map_err(EngineError::fs("capture restored paths"))?; + .await; + if let Err(failure) = captured { + if matches!( + failure.error, + acyclic_fs::CaptureError::RescanRequired { .. } + ) { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some(failure.error.to_string()); + self.reset_watch().await?; + self.baseline(CheckpointKind::Recovered).await?; + return Ok(()); + } + return Err(EngineError::fs("capture restored paths")(failure)); + } self.scrub_exclusions().await } diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 2b4c052..e46ddfa 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -6,10 +6,10 @@ use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -#[cfg(unix)] -use acyclic_fs::kernel::MetadataField; -use acyclic_fs::kernel::{FileKind, FilePayload, LogicalName, NamespacePath}; -use acyclic_fs::{materialize_checkout, ByteRange, MaterializeOptions}; +use acyclic_fs::kernel::{LogicalName, NamespacePath}; +use acyclic_fs::{ + materialize_checkout, materialize_checkout_host_path, MaterializeError, MaterializeOptions, +}; use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; use serde::{Deserialize, Serialize}; @@ -72,29 +72,59 @@ pub async fn restore_path_into( root: &Path, relative: &Path, ) -> Result { - let components = validate_relative(relative)?; + let mut checkout = store.checkout_exact(target).await?; + materialize_path_into_checkout(&mut checkout, root, relative, PathReplace::Atomic).await +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum PathReplace { + Atomic, + LiveMount, +} + +pub(crate) async fn materialize_path_into_checkout( + checkout: &mut LocalCheckout, + root: &Path, + relative: &Path, + replace: PathReplace, +) -> Result { + validate_relative(relative)?; + let normalized = normalized_relative(relative); let destination = root.join(relative); ensure_real_parents(root, relative)?; let parent = destination .parent() .ok_or_else(|| EngineError::Restore("path has no parent".into()))?; - let name = destination - .file_name() - .ok_or_else(|| EngineError::Restore("path has no name".into()))? - .to_string_lossy() - .into_owned(); - - let mut checkout = store.checkout_exact(target).await?; - let limits = checkout.volume_config().limits; let cancel = CancellationToken::new(); - let namespace = namespace_path(&components, limits)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - - let Some(record) = lookup.record else { + // The SDK requires an empty root and recreates the selected path below + // it. Keep ordinary restore staging outside the watched repository; + // live mounts must stage on the mount to permit the final rename. + let stage_parent = match replace { + PathReplace::Atomic => root + .parent() + .ok_or_else(|| EngineError::Restore("repository root has no parent".into()))?, + PathReplace::LiveMount => parent, + }; + let stage_root = create_restore_stage(stage_parent)?; + let staged = stage_root.join(&normalized); + let materialized = materialize_checkout_host_path( + checkout, + &normalized, + &MaterializeOptions { + destination: stage_root.clone(), + maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, + maximum_extent_spans: MAXIMUM_EXTENT_SPANS, + transfer_bytes: TRANSFER_BYTES, + }, + WorkCounters::UNBOUNDED, + &cancel, + ) + .await; + if matches!( + materialized.as_ref().map_err(|failure| &failure.error), + Err(MaterializeError::MissingPath) + ) { + std::fs::remove_dir_all(&stage_root)?; // Faithful restore of an absent path: remove it if it exists now. return match std::fs::symlink_metadata(&destination) { Ok(metadata) => { @@ -108,195 +138,109 @@ pub async fn restore_path_into( action: RestoreAction::Removed, }) } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - Err(EngineError::Restore(format!( + Err(error) if error.kind() == std::io::ErrorKind::NotFound => match replace { + PathReplace::Atomic => Err(EngineError::Restore(format!( "{} does not exist at that checkpoint or in the tree", relative.display() - ))) - } + ))), + PathReplace::LiveMount => Ok(RestoreOutcome { + path: relative.to_path_buf(), + action: RestoreAction::Removed, + }), + }, Err(error) => Err(error.into()), }; - }; - - // Stage next to the destination so the final rename never crosses a - // filesystem; the parent must exist (it did at the checkpoint, but the - // tree may have lost it since). - ensure_real_parents(root, relative)?; - let staged = parent.join(format!( - ".{name}.{}-restore-{}", - crate::product::NAME, - std::process::id() - )); - let _ = remove_any(&staged); - let written = write_node( - &mut checkout, - &namespace, - record.kind, - &record.payload, - &staged, - limits, - &cancel, - ) - .await; - if let Err(error) = written { - let _ = remove_any(&staged); - return Err(error); + } + if let Err(error) = materialized { + let _ = std::fs::remove_dir_all(&stage_root); + return Err(EngineError::Restore(format!("materialize path: {error}"))); } ensure_real_parents(root, relative)?; - // Swap in. An existing destination is exchanged atomically and the old - // node discarded; an absent one is a plain rename. + // A live mount must observe ordinary remove/rename operations through + // its driver. A user restore exchanges an existing node atomically. match std::fs::symlink_metadata(&destination) { - Ok(_) => { - if let Err(error) = atomic_exchange(&destination, &staged) { - let _ = remove_any(&staged); - return Err(error); + Ok(_) => match replace { + PathReplace::Atomic => { + // An exchange may have published the new node before a + // durability error. Keep both staged and scratch trees. + atomic_exchange(&destination, &staged)?; } - let _ = remove_any(&staged); - } + PathReplace::LiveMount => replace_live_mount(&staged, &destination, parent)?, + }, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - if let Err(error) = std::fs::rename(&staged, &destination) { - let _ = remove_any(&staged); + let renamed = match replace { + PathReplace::Atomic => durable_rename(&staged, &destination, false), + PathReplace::LiveMount => std::fs::rename(&staged, &destination), + }; + if let Err(error) = renamed { return Err(error.into()); } } Err(error) => { - let _ = remove_any(&staged); return Err(error.into()); } } + std::fs::remove_dir_all(&stage_root)?; + #[cfg(unix)] + if replace == PathReplace::Atomic { + sync_parent(&stage_root)?; + } Ok(RestoreOutcome { path: relative.to_path_buf(), action: RestoreAction::Restored, }) } -/// Writes one checkpoint node (recursively for directories) to a fresh host -/// path. Modes are applied; mtimes are not (same contract as a full rewind). -pub(crate) fn write_node<'a>( - checkout: &'a mut LocalCheckout, - namespace: &'a NamespacePath, - kind: FileKind, - payload: &'a FilePayload, - host: &'a Path, - limits: acyclic_fs::model::VolumeLimits, - cancel: &'a CancellationToken, -) -> std::pin::Pin> + 'a>> { - Box::pin(async move { - match kind { - FileKind::Regular => { - let length = match payload { - FilePayload::InlineRegular(inline) => inline.as_bytes().len() as u64, - FilePayload::Regular { logical_bytes, .. } => *logical_bytes, - _ => { - return Err(EngineError::Restore( - "regular file with foreign payload".into(), - )) - } - }; - let mut file = std::fs::File::create(host)?; - let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); - let mut offset = 0; - while offset < length { - let take = chunk.min(length - offset); - let read = checkout - .read_file_range( - namespace, - ByteRange { - offset, - length: take, - }, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("read file range"))? - .value; - use std::io::Write; - file.write_all(&read.bytes)?; - offset += take; - } - file.sync_all()?; - drop(file); - apply_mode(checkout, namespace, host, cancel).await - } - FileKind::SymbolicLink => { - let target = checkout - .read_symbolic_link(namespace, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("read symlink"))? - .value; - create_symlink(&target, host) - } - FileKind::Directory => { - std::fs::create_dir(host)?; - let mut entries = Vec::new(); - let mut after = None; - loop { - let page = checkout - .list_directory_records( - namespace, - after.as_ref(), - MAXIMUM_DIRECTORY_ENTRIES, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("list directory"))? - .value; - for entry in &page.entries { - entries.push((entry.name.clone(), entry.record.kind, entry.record.payload)); - } - match page.entries.last() { - Some(last) if page.has_more => after = Some(last.name.clone()), - _ => break, - } - } - for (name, kind, payload) in entries { - let mut components = namespace.components().to_vec(); - components.push(name.clone()); - let child = NamespacePath::new(components, limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}")))?; - let child_host = host.join(logical_to_os(&name)); - write_node( - checkout, - &child, - kind, - &payload, - &child_host, - limits, - cancel, - ) - .await?; - } - apply_mode(checkout, namespace, host, cancel).await - } - other => Err(EngineError::Restore(format!( - "cannot restore a {other:?} node (only files, symlinks, and directories)" - ))), +fn replace_live_mount(staged: &Path, destination: &Path, parent: &Path) -> Result<()> { + let backup_root = create_restore_stage(parent)?; + let backup = backup_root.join("old"); + if let Err(error) = std::fs::rename(destination, &backup) { + let _ = std::fs::remove_dir(&backup_root); + return Err(error.into()); + } + if let Err(error) = std::fs::rename(staged, destination) { + if let Err(rollback) = std::fs::rename(&backup, destination) { + return Err(EngineError::Restore(format!( + "materialize rename failed: {error}; old node remains at {} after rollback failed: {rollback}", + backup.display() + ))); } - }) + let _ = std::fs::remove_dir(&backup_root); + return Err(error.into()); + } + remove_any(&backup)?; + std::fs::remove_dir(&backup_root)?; + Ok(()) } -async fn apply_mode( - checkout: &mut LocalCheckout, - namespace: &NamespacePath, - host: &Path, - cancel: &CancellationToken, -) -> Result<()> { - let metadata = checkout - .read_metadata(namespace, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("read metadata"))? - .value; - #[cfg(unix)] - if let MetadataField::Value(mode) = metadata.posix_mode { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(host, std::fs::Permissions::from_mode(mode & 0o7777))?; +fn normalized_relative(relative: &Path) -> PathBuf { + relative + .components() + .filter_map(|component| match component { + Component::Normal(name) => Some(name), + _ => None, + }) + .collect() +} + +fn create_restore_stage(parent: &Path) -> Result { + for _ in 0..16 { + let sequence = PARK_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let stage = parent.join(format!( + ".{}-restore-{}-{sequence}", + crate::product::NAME, + std::process::id() + )); + match std::fs::create_dir(&stage) { + Ok(()) => return Ok(stage), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), + } } - #[cfg(not(unix))] - let _ = (metadata, host); - Ok(()) + Err(EngineError::Restore( + "could not allocate a unique restore stage".into(), + )) } pub(crate) fn validate_relative(relative: &Path) -> Result>> { @@ -396,22 +340,6 @@ fn os_to_bytes(name: &std::ffi::OsStr) -> Vec { crate::names::os_to_bytes(name) } -fn logical_to_os(name: &LogicalName) -> std::ffi::OsString { - crate::names::bytes_to_os(name.as_bytes()) -} - -#[cfg(unix)] -fn create_symlink(target: &[u8], host: &Path) -> Result<()> { - use std::os::unix::ffi::OsStrExt; - std::os::unix::fs::symlink(std::ffi::OsStr::from_bytes(target), host)?; - Ok(()) -} - -#[cfg(not(unix))] -fn create_symlink(_target: &[u8], _host: &Path) -> Result<()> { - Err(EngineError::Restore("symlink restore is unix-only".into())) -} - /// Crash-recovery journal. Present on disk only while a swap is in flight. #[derive(Debug, Serialize, Deserialize)] pub struct Journal { diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index 1df5441..12ab776 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -431,7 +431,19 @@ fn single_path_restore_leaves_the_rest_alone() { std::fs::write(root.join("src/main.rs"), b"v1\n").expect("seed"); std::fs::write(root.join("src/deep/a.txt"), b"a1\n").expect("seed"); std::fs::write(root.join("src/deep/b.txt"), b"b1\n").expect("seed"); + std::fs::write(root.join("src/deep/hard-a.txt"), b"linked\n").expect("seed hard link"); + std::fs::hard_link( + root.join("src/deep/hard-a.txt"), + root.join("src/deep/hard-b.txt"), + ) + .expect("hard link"); + std::fs::hard_link( + root.join("src/deep/hard-a.txt"), + root.join("outside-hard.txt"), + ) + .expect("hard link outside restored subtree"); std::fs::write(root.join("other.txt"), b"keep\n").expect("seed"); + std::fs::write(root.join("source.txt"), b"link source\n").expect("seed"); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; @@ -456,9 +468,15 @@ fn single_path_restore_leaves_the_rest_alone() { .expect("v1"); // Mutate everything. + // A new hard link can arrive as a lone watcher hint. The SDK must + // request a baseline instead of recording it as an independent file. + std::fs::hard_link(root.join("source.txt"), root.join("late-link.txt")) + .expect("late hard link"); std::fs::write(root.join("src/main.rs"), b"v2\n").expect("edit"); std::fs::write(root.join("src/deep/a.txt"), b"a2\n").expect("edit"); std::fs::remove_file(root.join("src/deep/b.txt")).expect("rm"); + std::fs::remove_file(root.join("src/deep/hard-b.txt")).expect("rm hard link"); + std::fs::write(root.join("src/deep/hard-a.txt"), b"changed\n").expect("edit hard link"); std::fs::write(root.join("src/deep/c.txt"), b"c2\n").expect("add"); std::fs::write(root.join("other.txt"), b"changed\n").expect("edit"); std::fs::write(root.join("new.txt"), b"new\n").expect("add"); @@ -514,6 +532,16 @@ fn single_path_restore_leaves_the_rest_alone() { b"b1\n" ); assert!(!root.join("src/deep/c.txt").exists()); + std::fs::write(root.join("src/deep/hard-a.txt"), b"relinked\n") + .expect("write restored hard link"); + assert_eq!( + std::fs::read(root.join("src/deep/hard-b.txt")).expect("read restored hard link"), + b"relinked\n" + ); + assert_eq!( + std::fs::read(root.join("outside-hard.txt")).expect("read untouched outside link"), + b"changed\n" + ); assert_eq!( std::fs::read(root.join("other.txt")).expect("read"), b"changed\n" From 3d48d192776ca8e2516b7a2eda35b2bc483d0e46 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 21:12:03 +0100 Subject: [PATCH 019/106] Keep checkpoint roles in plugin and serialize timeline writes --- crates/acyclic/src/checkpoint_kind.rs | 133 +++++++++++++ crates/acyclic/src/index.rs | 273 +++++++++++++++----------- crates/acyclic/src/lib.rs | 1 + 3 files changed, 297 insertions(+), 110 deletions(-) create mode 100644 crates/acyclic/src/checkpoint_kind.rs diff --git a/crates/acyclic/src/checkpoint_kind.rs b/crates/acyclic/src/checkpoint_kind.rs new file mode 100644 index 0000000..2b7ab28 --- /dev/null +++ b/crates/acyclic/src/checkpoint_kind.rs @@ -0,0 +1,133 @@ +//! Checkpoint roles stored by the plugin's timeline index. + +use serde::{Deserialize, Serialize}; + +/// Why a checkpoint observation exists. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CheckpointKind { + /// First complete observation of a source. + Baseline, + /// State immediately before a consumer operation. + Pre, + /// State immediately after a consumer operation. + Post, + /// Consumer-requested observation. + Manual, + /// Safety observation before moving the live head backward. + PreRewind, + /// Observation created while recovering a source. + Recovered, + /// Failed capture; its generation belongs to an earlier observation. + Failed, + /// A request whose state did not change. + Noop, + /// Observation created by an idle timer. + Auto, +} + +impl CheckpointKind { + /// SQL filter for user-facing rewind targets. Keep this beside the role + /// definition so timeline queries do not each repeat the list. + pub const TARGETS_SQL: &'static str = "'baseline','pre','post','manual','auto'"; + + /// Stable value used by existing timeline stores and wire consumers. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Baseline => "baseline", + Self::Pre => "pre", + Self::Post => "post", + Self::Manual => "manual", + Self::PreRewind => "pre_rewind", + Self::Recovered => "recovered", + Self::Failed => "failed", + Self::Noop => "noop", + Self::Auto => "auto", + } + } + + /// The row's generation is a real state that may be restored. + #[must_use] + pub const fn is_restorable(self) -> bool { + !matches!(self, Self::Failed) + } + + /// This observation is a user-facing rewind target and branch member. + #[must_use] + pub const fn is_target(self) -> bool { + matches!( + self, + Self::Baseline | Self::Pre | Self::Post | Self::Manual | Self::Auto + ) + } +} + +impl std::str::FromStr for CheckpointKind { + type Err = UnknownCheckpointKind; + + fn from_str(value: &str) -> Result { + match value { + "baseline" => Ok(Self::Baseline), + "pre" => Ok(Self::Pre), + "post" => Ok(Self::Post), + "manual" => Ok(Self::Manual), + "pre_rewind" => Ok(Self::PreRewind), + "recovered" => Ok(Self::Recovered), + "failed" => Ok(Self::Failed), + "noop" => Ok(Self::Noop), + "auto" => Ok(Self::Auto), + _ => Err(UnknownCheckpointKind(value.to_owned())), + } + } +} + +impl std::fmt::Display for CheckpointKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.pad(self.as_str()) + } +} + +/// A timeline store contained an unsupported checkpoint role. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +#[error("unknown checkpoint kind {0}")] +pub struct UnknownCheckpointKind(pub String); + +#[cfg(test)] +mod tests { + use super::CheckpointKind; + + #[test] + fn storage_values_and_restore_roles_are_stable() { + let roles = [ + ("baseline", true, true), + ("pre", true, true), + ("post", true, true), + ("manual", true, true), + ("pre_rewind", true, false), + ("recovered", true, false), + ("failed", false, false), + ("noop", true, false), + ("auto", true, true), + ]; + for (value, restorable, target) in roles { + let Ok(kind) = value.parse::() else { + unreachable!("known checkpoint role"); + }; + assert_eq!(kind.as_str(), value); + assert_eq!(kind.is_restorable(), restorable); + assert_eq!(kind.is_target(), target); + } + assert!("future".parse::().is_err()); + let sql_roles = CheckpointKind::TARGETS_SQL + .split(',') + .map(|role| { + role.trim_matches('\'') + .parse::() + .expect("SQL role") + }) + .collect::>(); + assert_eq!(sql_roles.len(), 5); + assert!(sql_roles.iter().all(|role| role.is_target())); + } +} diff --git a/crates/acyclic/src/index.rs b/crates/acyclic/src/index.rs index d55bbd7..6d3a196 100644 --- a/crates/acyclic/src/index.rs +++ b/crates/acyclic/src/index.rs @@ -7,11 +7,11 @@ use std::path::Path; +pub use crate::checkpoint_kind::CheckpointKind; use acyclic_fs::{Digest, GenerationId}; use rusqlite::{params, Connection, OptionalExtension}; -use serde::{Deserialize, Serialize}; -use crate::{EngineError, Result}; +use crate::Result; /// One checkpoint row. #[derive(Clone, Debug, PartialEq)] @@ -70,65 +70,7 @@ impl CheckpointRow { /// `failed` rows carry the generation from BEFORE the failed capture — /// restoring one would claim a state the row does not represent. pub fn is_restorable(&self) -> bool { - self.kind != CheckpointKind::Failed - } -} - -/// Why a checkpoint exists. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CheckpointKind { - Baseline, - Pre, - Post, - Manual, - PreRewind, - Recovered, - Failed, - Noop, - /// Taken by the idle timer, not any request: the safety net for hosts - /// with no lifecycle-hook API (see `Pipeline::auto_checkpoint`). - Auto, -} - -impl CheckpointKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Baseline => "baseline", - Self::Pre => "pre", - Self::Post => "post", - Self::Manual => "manual", - Self::PreRewind => "pre_rewind", - Self::Recovered => "recovered", - Self::Failed => "failed", - Self::Noop => "noop", - Self::Auto => "auto", - } - } - - fn parse(text: &str) -> Result { - Ok(match text { - "baseline" => Self::Baseline, - "pre" => Self::Pre, - "post" => Self::Post, - "manual" => Self::Manual, - "pre_rewind" => Self::PreRewind, - "recovered" => Self::Recovered, - "failed" => Self::Failed, - "noop" => Self::Noop, - "auto" => Self::Auto, - other => { - return Err(EngineError::Store(format!( - "unknown checkpoint kind {other}" - ))) - } - }) - } -} - -impl std::fmt::Display for CheckpointKind { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.pad(self.as_str()) + self.kind.is_restorable() } } @@ -206,11 +148,14 @@ impl Index { kind: CheckpointKind, attribution: &Attribution, ) -> Result { - let turn = self.effective_turn(attribution)?; + let transaction = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; if let Some(session_id) = attribution.session_id.as_deref() { - self.ensure_session(session_id)?; + ensure_session_on(&transaction, session_id)?; } - self.connection.execute( + let turn = effective_turn_on(&transaction, attribution)?; + transaction.execute( "INSERT INTO checkpoints (generation, created_at, kind, session_id, tool_call_id, tool_name, label, turn, rewind_target) @@ -227,25 +172,9 @@ impl Index { attribution.rewind_target, ], )?; - Ok(self.connection.last_insert_rowid()) - } - - /// The turn a new checkpoint belongs to: the explicit one, else the - /// session's latest recorded turn (hooks don't know turn numbers; the - /// `UserPromptSubmit` hook records them and tool hooks inherit). - fn effective_turn(&self, attribution: &Attribution) -> Result> { - if attribution.turn.is_some() { - return Ok(attribution.turn); - } - let Some(session_id) = attribution.session_id.as_deref() else { - return Ok(None); - }; - let turn: Option = self.connection.query_row( - "SELECT MAX(turn) FROM turns WHERE session_id = ?1", - params![session_id], - |row| row.get(0), - )?; - Ok(turn) + let id = transaction.last_insert_rowid(); + transaction.commit()?; + Ok(id) } /// Records a failed capture attempt (no generation advanced: the previous @@ -256,8 +185,14 @@ impl Index { error: &str, attribution: &Attribution, ) -> Result { - let turn = self.effective_turn(attribution)?; - self.connection.execute( + let transaction = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + if let Some(session_id) = attribution.session_id.as_deref() { + ensure_session_on(&transaction, session_id)?; + } + let turn = effective_turn_on(&transaction, attribution)?; + transaction.execute( "INSERT INTO checkpoints (generation, created_at, kind, session_id, tool_call_id, tool_name, error, turn) VALUES (?1, ?2, 'failed', ?3, ?4, ?5, ?6, ?7)", @@ -271,7 +206,9 @@ impl Index { turn, ], )?; - Ok(self.connection.last_insert_rowid()) + let id = transaction.last_insert_rowid(); + transaction.commit()?; + Ok(id) } /// Marks every checkpoint up to `through_id` as covered by an authority @@ -315,8 +252,9 @@ impl Index { .query_row( &format!( "SELECT {CHECKPOINT_COLUMNS} - FROM checkpoints WHERE kind IN ('baseline','pre','post','manual','auto') - ORDER BY id DESC LIMIT 1" + FROM checkpoints WHERE kind IN ({}) + ORDER BY id DESC LIMIT 1", + CheckpointKind::TARGETS_SQL ), [], row_to_checkpoint, @@ -332,8 +270,9 @@ impl Index { .query_row( &format!( "SELECT {CHECKPOINT_COLUMNS} FROM checkpoints - WHERE id < ?1 AND kind IN ('baseline','pre','post','manual','auto') - ORDER BY id DESC LIMIT 1" + WHERE id < ?1 AND kind IN ({}) + ORDER BY id DESC LIMIT 1", + CheckpointKind::TARGETS_SQL ), params![id], row_to_checkpoint, @@ -435,8 +374,9 @@ impl Index { .query_row( &format!( "SELECT {CHECKPOINT_COLUMNS} FROM checkpoints - WHERE session_id = ?1 AND kind IN ('baseline','pre','post','manual','auto') - ORDER BY id DESC LIMIT 1" + WHERE session_id = ?1 AND kind IN ({}) + ORDER BY id DESC LIMIT 1", + CheckpointKind::TARGETS_SQL ), params![session_id], row_to_checkpoint, @@ -464,8 +404,9 @@ impl Index { pub fn between(&self, after_id: i64, before_id: i64) -> Result> { let mut statement = self.connection.prepare(&format!( "SELECT {CHECKPOINT_COLUMNS} FROM checkpoints - WHERE id > ?1 AND id < ?2 AND kind IN ('baseline','pre','post','manual','auto') - ORDER BY id ASC" + WHERE id > ?1 AND id < ?2 AND kind IN ({}) + ORDER BY id ASC", + CheckpointKind::TARGETS_SQL ))?; let rows = statement.query_map(params![after_id, before_id], row_to_checkpoint)?; rows.map(|row| row.map_err(Into::into)).collect() @@ -475,15 +416,19 @@ impl Index { /// The prompt is truncated to an excerpt on a char boundary. pub fn turn_started(&mut self, session_id: &str, prompt: &str) -> Result { self.ensure_session(session_id)?; - let next: i64 = self.connection.query_row( + let transaction = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let next: i64 = transaction.query_row( "SELECT COALESCE(MAX(turn), 0) + 1 FROM turns WHERE session_id = ?1", params![session_id], |row| row.get(0), )?; - self.connection.execute( + transaction.execute( "INSERT INTO turns(session_id, turn, started_at, prompt) VALUES (?1, ?2, ?3, ?4)", params![session_id, next, now(), excerpt(prompt)], )?; + transaction.commit()?; Ok(next) } @@ -580,11 +525,7 @@ impl Index { /// The host is unknown at this point; a later `session_started` is /// ignored by the primary key, so the row keeps its earliest start. fn ensure_session(&mut self, session_id: &str) -> Result<()> { - self.connection.execute( - "INSERT OR IGNORE INTO sessions(session_id, host, started_at) VALUES (?1, NULL, ?2)", - params![session_id, now()], - )?; - Ok(()) + ensure_session_on(&self.connection, session_id) } pub fn session_ended(&mut self, session_id: &str) -> Result<()> { @@ -596,6 +537,32 @@ impl Index { } } +fn ensure_session_on(connection: &Connection, session_id: &str) -> Result<()> { + connection.execute( + "INSERT OR IGNORE INTO sessions(session_id, host, started_at) VALUES (?1, NULL, ?2)", + params![session_id, now()], + )?; + Ok(()) +} + +/// Use an explicit turn when supplied; otherwise inherit the session's most +/// recent turn. Checkpoint insertion and this lookup share one write transaction. +fn effective_turn_on(connection: &Connection, attribution: &Attribution) -> Result> { + if attribution.turn.is_some() { + return Ok(attribution.turn); + } + let Some(session_id) = attribution.session_id.as_deref() else { + return Ok(None); + }; + connection + .query_row( + "SELECT MAX(turn) FROM turns WHERE session_id = ?1", + params![session_id], + |row| row.get(0), + ) + .map_err(Into::into) +} + fn row_to_checkpoint(row: &rusqlite::Row<'_>) -> rusqlite::Result { let corrupt = |message: &str| { rusqlite::Error::FromSqlConversionFailure( @@ -614,7 +581,9 @@ fn row_to_checkpoint(row: &rusqlite::Row<'_>) -> rusqlite::Result id: row.get(0)?, generation: GenerationId::new(Digest::from_bytes(bytes)), created_at: row.get(2)?, - kind: CheckpointKind::parse(&kind_text).map_err(|error| corrupt(&error.to_string()))?, + kind: kind_text + .parse::() + .map_err(|error| corrupt(&error.to_string()))?, published: row.get::<_, i64>(4)? != 0, session_id: row.get(5)?, tool_call_id: row.get(6)?, @@ -703,9 +672,19 @@ fn ensure_auto_kind_allowed(connection: &Connection) -> Result<()> { if sql.contains("'auto'") { return Ok(()); } - connection.execute_batch( - "BEGIN IMMEDIATE; - ALTER TABLE checkpoints RENAME TO checkpoints_pre_auto; + let saved_schema = connection + .prepare( + "SELECT type, sql FROM sqlite_master + WHERE tbl_name = 'checkpoints' AND type IN ('index', 'trigger') AND sql IS NOT NULL + ORDER BY CASE type WHEN 'index' THEN 0 ELSE 1 END, name", + )? + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::>>()?; + let transaction = connection.unchecked_transaction()?; + transaction.execute_batch( + "ALTER TABLE checkpoints RENAME TO checkpoints_pre_auto; CREATE TABLE checkpoints( id INTEGER PRIMARY KEY, generation BLOB NOT NULL, @@ -727,10 +706,16 @@ fn ensure_auto_kind_allowed(connection: &Connection) -> Result<()> { tool_name, label, error, turn, rewind_target FROM checkpoints_pre_auto; DROP TABLE checkpoints_pre_auto; - CREATE INDEX IF NOT EXISTS checkpoints_by_generation ON checkpoints(generation); - CREATE INDEX IF NOT EXISTS checkpoints_by_session ON checkpoints(session_id, id); - COMMIT;", + ", + )?; + for (_, definition) in saved_schema { + transaction.execute_batch(&definition)?; + } + transaction.execute_batch( + "CREATE INDEX IF NOT EXISTS checkpoints_by_generation ON checkpoints(generation); + CREATE INDEX IF NOT EXISTS checkpoints_by_session ON checkpoints(session_id, id);", )?; + transaction.commit()?; Ok(()) } @@ -891,6 +876,48 @@ mod tests { assert!(late.started_at <= started); } + #[test] + fn concurrent_turn_writers_get_distinct_numbers() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("index.db"); + let mut first = Index::open(&path).expect("first index"); + let mut second = Index::open(&path).expect("second index"); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let other_barrier = barrier.clone(); + let other = std::thread::spawn(move || { + other_barrier.wait(); + second.turn_started("shared", "second") + }); + barrier.wait(); + let first_turn = first.turn_started("shared", "first").expect("first turn"); + let second_turn = other.join().expect("writer thread").expect("second turn"); + assert_ne!(first_turn, second_turn); + assert_eq!(first.turns(Some("shared")).expect("turns").len(), 2); + } + + #[test] + fn failed_capture_backfills_its_session_and_inherits_the_latest_turn() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut index = Index::open(&dir.path().join("index.db")).expect("open"); + let attribution = Attribution { + session_id: Some("missed-start".into()), + ..Attribution::default() + }; + let first = index + .record_failure(generation(1), "capture failed", &attribution) + .expect("first failed capture"); + assert_eq!(index.by_id(first).expect("query").expect("row").turn, None); + assert_eq!(index.sessions(10).expect("sessions").len(), 1); + index.turn_started("missed-start", "retry").expect("turn"); + let second = index + .record_failure(generation(1), "capture failed again", &attribution) + .expect("second failed capture"); + assert_eq!( + index.by_id(second).expect("query").expect("row").turn, + Some(1) + ); + } + #[test] fn turns_link_checkpoints_and_resolve_both_ways() { let dir = tempfile::tempdir().expect("tempdir"); @@ -1039,7 +1066,9 @@ mod tests { published INTEGER NOT NULL DEFAULT 0, session_id TEXT, tool_call_id TEXT, tool_name TEXT, label TEXT, error TEXT); INSERT INTO checkpoints(generation, created_at, kind) - VALUES (zeroblob(32), 1, 'post');", + VALUES (zeroblob(32), 1, 'post'); + CREATE INDEX checkpoints_by_generation ON checkpoints(generation); + CREATE INDEX checkpoints_by_session ON checkpoints(session_id, id);", ) .expect("seed"); } @@ -1070,7 +1099,11 @@ mod tests { tool_call_id TEXT, tool_name TEXT, label TEXT, error TEXT, turn INTEGER, rewind_target INTEGER); INSERT INTO checkpoints(generation, created_at, kind) - VALUES (zeroblob(32), 1, 'post');", + VALUES (zeroblob(32), 1, 'post'); + CREATE TABLE checkpoint_audit(inserted INTEGER NOT NULL); + CREATE INDEX custom_post_idx ON checkpoints(created_at) WHERE kind = 'post'; + CREATE TRIGGER audit_checkpoint_insert AFTER INSERT ON checkpoints + BEGIN INSERT INTO checkpoint_audit(inserted) VALUES (NEW.id); END;", ) .expect("seed"); } @@ -1086,5 +1119,25 @@ mod tests { .expect("auto checkpoint should now be a valid kind"); let row = index.by_id(row_id).expect("q").expect("row"); assert_eq!(row.kind, CheckpointKind::Auto); + let indexes: Vec = index + .connection + .prepare("PRAGMA index_list(checkpoints)") + .expect("prepare index list") + .query_map([], |row| row.get(1)) + .expect("index list") + .collect::>() + .expect("index names"); + assert!(indexes + .iter() + .any(|name| name == "checkpoints_by_generation")); + assert!(indexes.iter().any(|name| name == "checkpoints_by_session")); + assert!(indexes.iter().any(|name| name == "custom_post_idx")); + let audit_id: i64 = index + .connection + .query_row("SELECT inserted FROM checkpoint_audit", [], |row| { + row.get(0) + }) + .expect("custom trigger fired after migration"); + assert_eq!(audit_id, row_id); } } diff --git a/crates/acyclic/src/lib.rs b/crates/acyclic/src/lib.rs index 6c03408..69f7e97 100644 --- a/crates/acyclic/src/lib.rs +++ b/crates/acyclic/src/lib.rs @@ -40,6 +40,7 @@ pub fn short_hex(hex: &str) -> &str { hex.get(..12).unwrap_or(hex) } +pub mod checkpoint_kind; pub mod config; pub mod diff; pub mod exclude; From 82a4d92ab28919d0370daafe8d7aa1180fc2b776 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Fri, 18 Sep 2026 21:16:26 +0100 Subject: [PATCH 020/106] Bound prompt excerpt work to stored length --- crates/acyclic/src/index.rs | 84 +++++++++++++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/crates/acyclic/src/index.rs b/crates/acyclic/src/index.rs index 6d3a196..32ecd6b 100644 --- a/crates/acyclic/src/index.rs +++ b/crates/acyclic/src/index.rs @@ -721,16 +721,45 @@ fn ensure_auto_kind_allowed(connection: &Connection) -> Result<()> { /// Bounded, single-line prompt excerpt: whitespace runs collapsed, cut on a /// char boundary at [`PROMPT_EXCERPT_BYTES`] with an ellipsis. +/// +/// The original prompt is scanned only until the excerpt fills. pub fn excerpt(prompt: &str) -> String { - let collapsed = prompt.split_whitespace().collect::>().join(" "); - if collapsed.len() <= PROMPT_EXCERPT_BYTES { - return collapsed; - } - let mut cut = PROMPT_EXCERPT_BYTES - 1; - while !collapsed.is_char_boundary(cut) { - cut -= 1; + let mut collapsed = String::with_capacity(PROMPT_EXCERPT_BYTES); + let mut words = prompt.split_whitespace().peekable(); + while let Some(word) = words.next() { + let separator = usize::from(!collapsed.is_empty()); + let remaining = PROMPT_EXCERPT_BYTES.saturating_sub(collapsed.len() + separator); + if word.len() > remaining { + let target = PROMPT_EXCERPT_BYTES - 1; + if separator != 0 && collapsed.len() < target { + collapsed.push(' '); + } + if word.len() >= target.saturating_sub(collapsed.len()) { + for character in word.chars() { + if collapsed.len() + character.len_utf8() > target { + break; + } + collapsed.push(character); + } + } else { + collapsed.push_str(word); + } + collapsed.push('…'); + return collapsed; + } + if separator != 0 { + collapsed.push(' '); + } + collapsed.push_str(word); + if collapsed.len() == PROMPT_EXCERPT_BYTES && words.peek().is_some() { + if let Some((index, _)) = collapsed.char_indices().next_back() { + collapsed.truncate(index); + } + collapsed.push('…'); + return collapsed; + } } - format!("{}…", collapsed.get(..cut).unwrap_or(&collapsed)) + collapsed } fn now() -> i64 { @@ -1050,6 +1079,45 @@ mod tests { assert!(cut.len() <= PROMPT_EXCERPT_BYTES + "…".len()); assert!(cut.ends_with('…')); assert_eq!(excerpt("a b\n\tc"), "a b c"); + let exact = "x".repeat(PROMPT_EXCERPT_BYTES); + assert_eq!(excerpt(&exact), exact); + assert_eq!( + excerpt(&format!("{exact} tail")), + format!("{}…", "x".repeat(PROMPT_EXCERPT_BYTES - 1)) + ); + let boundary = format!("{} é", "x".repeat(PROMPT_EXCERPT_BYTES - 2)); + assert_eq!( + excerpt(&boundary), + format!("{} …", "x".repeat(PROMPT_EXCERPT_BYTES - 2)) + ); + } + + #[test] + fn excerpt_matches_the_full_normalization_model() { + fn full(prompt: &str) -> String { + let collapsed = prompt.split_whitespace().collect::>().join(" "); + if collapsed.len() <= PROMPT_EXCERPT_BYTES { + return collapsed; + } + let mut cut = PROMPT_EXCERPT_BYTES - 1; + while !collapsed.is_char_boundary(cut) { + cut -= 1; + } + format!("{}…", collapsed.get(..cut).expect("character boundary")) + } + + let atoms = ["a", " ", "\n", "é", "☃", "𐍈", " ", "xyz"]; + let mut state = 0x9e37_79b9_7f4a_7c15_u64; + for _ in 0..1_000 { + let mut prompt = String::new(); + for _ in 0..400 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + prompt.push_str(atoms[(state as usize) % atoms.len()]); + } + assert_eq!(excerpt(&prompt), full(&prompt)); + } } #[test] From 472401207f8cb2cf5d3dcc27e12acbe5da04a675 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 00:56:07 +0100 Subject: [PATCH 021/106] Cut plugin hot paths and harden Windows workflows --- Cargo.lock | 21 +++++++++ crates/acyclic/src/fork.rs | 28 ++++-------- crates/acyclic/src/index.rs | 14 ++---- crates/acyclic/src/main.rs | 14 ++++-- crates/acyclic/src/pipeline.rs | 35 +++++++++------ crates/acyclic/src/server.rs | 72 +++++++++++++++---------------- tests/acceptance/windows-smoke.sh | 53 ++++++++++++++++++++--- 7 files changed, 149 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 953be2a..9bebc90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,7 @@ dependencies = [ name = "acyclic-fs" version = "0.2.0-rc.5" dependencies = [ + "acyclic-linux-io", "acyclic-objects", "acyclic-stream", "async-trait", @@ -56,10 +57,19 @@ dependencies = [ "windows", ] +[[package]] +name = "acyclic-linux-io" +version = "0.1.0-rc.1" +dependencies = [ + "io-uring", + "libc", +] + [[package]] name = "acyclic-objects" version = "1.0.0-rc.4" dependencies = [ + "acyclic-linux-io", "async-trait", "blake3", "bytes", @@ -1040,6 +1050,17 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" +[[package]] +name = "io-uring" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.12.1" diff --git a/crates/acyclic/src/fork.rs b/crates/acyclic/src/fork.rs index a3df150..94c9fe7 100644 --- a/crates/acyclic/src/fork.rs +++ b/crates/acyclic/src/fork.rs @@ -46,8 +46,8 @@ impl ForkMode { } } -/// What the host can do for fork and Safe Mode mounts, probed once at -/// daemon start and reported by `status`/`init`. +/// What the host can do for fork mounts, probed once at daemon start and +/// reported by `status`/`init`. #[derive(Clone, Debug)] pub struct MountCapability { /// Human name of the provider this build would use ("fuse", "nfs loopback"). @@ -69,16 +69,8 @@ impl MountCapability { /// Live probe of the native mount provider. /// -/// Windows is held to copy forks whatever the probe says. `ProjFS` mounts -/// and projects correctly there — a fork's tree appears and reads back fine — -/// but writes into the projection stop at the `ProjFS` local cache and never -/// reach the overlay checkout, so `fork-diff` reports no changes and -/// `promote` lands nothing. Silently discarding a fork's work is far worse -/// than copying it, and copy forks are verified on Windows: write, diff and -/// promote all behave. -/// -/// Revisit when the sdk's `ProjFS` provider carries writes back. The probe -/// still runs so `status` can name the provider it found. +/// Windows forks use copies until `ProjFS` can capture every admitted host +/// mutation, including imports from outside its virtualized namespace. pub fn mount_capability() -> MountCapability { let probe = probe_native_mount(); let provider = match probe.kind { @@ -94,8 +86,7 @@ pub fn mount_capability() -> MountCapability { provider, available: false, reason: Some( - "ProjFS projects a fork but does not carry writes back to the store, \ - so forks use full copies (promote works the same)" + "ProjFS cannot capture every external import into a writable fork; forks use copies" .to_owned(), ), } @@ -129,11 +120,10 @@ pub fn mount_setup_hint() -> &'static str { ) } else if cfg!(windows) { concat!( - "forks use full copies on Windows. ProjFS can project a fork, but it does\n", - "not carry writes back to the store, so a mounted fork would silently lose\n", - "your work; copies land correctly through `promote`. Nothing to install.\n", - "Safe Mode (dry_run) needs mounts, so it is unavailable on Windows for the\n", - "same reason.", + "forks use full copies on Windows. ProjFS cannot capture every\n", + "external import into a writable fork, so mounted forks are not\n", + "admitted. Safe Mode (dry_run) cannot shadow an existing Windows\n", + "directory with ProjFS.", ) } else { "native mounts are not supported on this platform; forks use full copies \ diff --git a/crates/acyclic/src/index.rs b/crates/acyclic/src/index.rs index 32ecd6b..c3f8994 100644 --- a/crates/acyclic/src/index.rs +++ b/crates/acyclic/src/index.rs @@ -415,10 +415,10 @@ impl Index { /// Records the start of a conversation turn; returns its 1-based number. /// The prompt is truncated to an excerpt on a char boundary. pub fn turn_started(&mut self, session_id: &str, prompt: &str) -> Result { - self.ensure_session(session_id)?; let transaction = self .connection .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + ensure_session_on(&transaction, session_id)?; let next: i64 = transaction.query_row( "SELECT COALESCE(MAX(turn), 0) + 1 FROM turns WHERE session_id = ?1", params![session_id], @@ -519,15 +519,6 @@ impl Index { Ok(()) } - /// A turn or checkpoint for a session the daemon never saw start (its - /// `SessionStart` hook fired while the daemon was down) still gets a - /// session row, so `sessions`, `diff --turn`, and `brief` can find it. - /// The host is unknown at this point; a later `session_started` is - /// ignored by the primary key, so the row keeps its earliest start. - fn ensure_session(&mut self, session_id: &str) -> Result<()> { - ensure_session_on(&self.connection, session_id) - } - pub fn session_ended(&mut self, session_id: &str) -> Result<()> { self.connection.execute( "UPDATE sessions SET ended_at = ?2 WHERE session_id = ?1", @@ -538,6 +529,9 @@ impl Index { } fn ensure_session_on(connection: &Connection, session_id: &str) -> Result<()> { + // A turn or checkpoint for a session whose SessionStart hook was missed + // still needs a discoverable session row. A later session_started call + // fills in its host without replacing this earlier start time. connection.execute( "INSERT OR IGNORE INTO sessions(session_id, host, started_at) VALUES (?1, NULL, ?2)", params![session_id, now()], diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index 244ba75..fa7bb69 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -354,7 +354,7 @@ fn run(cli: Cli, repo: &Path) -> i32 { return 1; } }; - match execute(&mut client, command) { + match execute(&mut client, command, repo) { Ok(()) => 0, Err(message) => { eprintln!("{}: {message}", product::NAME); @@ -591,7 +591,7 @@ fn init(repo: &Path) -> i32 { clippy::too_many_lines, reason = "one arm per Command; each arm is a single call plus its printing" )] -fn execute(client: &mut Client, command: Command) -> Result<(), String> { +fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), String> { match command { Command::Checkpoint { message, @@ -1033,7 +1033,15 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { } Command::Stop => { client.call(proto::Op::Stop)?; - println!("daemon stopping"); + let pidfile = store_paths(repo)?.pidfile(); + let started = std::time::Instant::now(); + while pidfile.exists() { + if started.elapsed() >= std::time::Duration::from_secs(30) { + return Err("daemon did not finish stopping within 30 seconds".into()); + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + println!("daemon stopped"); Ok(()) } Command::SessionStart { session_id, host } => { diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 809979b..b61fb80 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -1440,14 +1440,18 @@ impl Pipeline { }) } - /// Polls the watcher until it stays quiet for `quiesce_ms` (capped at - /// `quiesce_cap_ms`), capturing every non-empty batch. Returns whether - /// anything was captured. + /// Captures every available watcher batch, then waits for `quiesce_ms` + /// of quiet after the first change (capped at `quiesce_cap_ms`). An empty + /// first poll returns immediately. Returns whether anything was captured. + #[allow( + clippy::too_many_lines, + reason = "watch invalidation and capture share one poll loop" + )] async fn drain_watcher(&mut self) -> Result { let quiesce = Duration::from_millis(self.config.quiesce_ms); let cap = Duration::from_millis(self.config.quiesce_cap_ms); let started = Instant::now(); - let mut last_change = Instant::now(); + let mut last_change = None; let mut changed = false; let mut polls = 0u32; let mut batches = 0u32; @@ -1510,16 +1514,21 @@ impl Pipeline { self.scrub_exclusions().await?; } changed = true; - last_change = Instant::now(); + last_change = Some(Instant::now()); } WatchBatch::Changes { .. } => { - if last_change.elapsed() >= quiesce || started.elapsed() >= cap { + if last_change.is_none() + || last_change.is_some_and(|last| last.elapsed() >= quiesce) + || started.elapsed() >= cap + { crate::trace!( "pipeline", "drain: done after {:.1}ms, {polls} polls, {batches} batch(es), \ {hints} hint(s); stopped by {}", crate::trace::ms(started), - if started.elapsed() >= cap { + if last_change.is_none() { + "empty poll" + } else if started.elapsed() >= cap { "cap" } else { "quiesce window" @@ -1623,12 +1632,12 @@ impl Pipeline { /// The safety net for hosts with no lifecycle-hook API (Claude Desktop /// over MCP): a checkpoint no request asked for, taken once the watcher /// has been quiet for `auto_checkpoint_idle_ms`. Runs on every idle - /// tick: drains whatever the watcher has (cheap, bounded by - /// `quiesce_ms`; usually nothing, since hook-driven hosts drain on their - /// own pre/post-tool checkpoints), then records a row only once a full - /// tick has passed with no further changes and nothing else has - /// checkpointed them in the meantime. Never records a row when nothing - /// changed, so it cannot spam the timeline. + /// tick: drains whatever the watcher has (an empty watcher returns after + /// one poll; hook-driven hosts usually drained on their own pre/post-tool + /// checkpoints), then records a row only once a full tick has passed with + /// no further changes and nothing else has checkpointed them in the + /// meantime. Never records a row when nothing changed, so it cannot spam + /// the timeline. async fn auto_checkpoint(&mut self) { if self.config.auto_checkpoint_idle_ms == 0 || self.shadowed || self.state != State::Ready { return; diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 4584b41..efdf1a5 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -956,30 +956,12 @@ impl Server { } } proto::Op::ForkDiff { id } => { - let (base, shared, copy_dir) = { + let (base, shared, capture_dir) = { let forks = self.forks.lock().await; let fork = forks.get(&id).ok_or(format!("no fork {id}"))?; (fork.base, Arc::clone(&fork.shared), fork.copy_dir.clone()) }; - // Mounted fork: its writes already sit in its own overlay, so - // snapshot that. Copy fork: read the directory into a - // scratch overlay pinned at the base (native capture cannot - // read through the mount itself: NFS lacks the extent ioctl). - // Either way the fork stays promotable afterwards. - let overlay = match copy_dir { - None => shared, - Some(dir) => { - let scratch = self - .handle - .scratch_checkout(base) - .await - .map_err(stringify)?; - fork::capture_copy(&scratch, &dir) - .await - .map_err(stringify)?; - scratch - } - }; + let overlay = self.fork_view(base, shared, capture_dir.as_deref()).await?; let changes = if overlay.lock().await.has_pending_mutations() { let generation = self .handle @@ -1205,6 +1187,34 @@ impl Server { Ok(proto::Reply::Forks(created)) } + /// Uses the live overlay for mounted forks. Copy forks are reconciled + /// from the host tree into a scratch checkout before diff or promotion. + async fn fork_view( + &self, + base: acyclic::GenerationId, + shared: Arc, + capture_dir: Option<&Path>, + ) -> Result, String> { + let Some(dir) = capture_dir else { + return Ok(shared); + }; + let started = std::time::Instant::now(); + let scratch = self + .handle + .scratch_checkout(base) + .await + .map_err(stringify)?; + let scratch_ms = acyclic::trace::ms(started); + let capture_started = std::time::Instant::now(); + fork::capture_copy(&scratch, dir).await.map_err(stringify)?; + acyclic::trace!( + "daemon", + "copy fork capture: scratch {scratch_ms:.1}ms, full-tree capture {:.1}ms", + acyclic::trace::ms(capture_started) + ); + Ok(scratch) + } + /// Lands a fork in place. The fork's paths are merged onto the current /// head (a three-way merge when the mainline moved; a plain write of /// the fork's paths when it did not) and written onto the real tree @@ -1217,24 +1227,10 @@ impl Server { fork: &ForkState, label: &str, ) -> Result { - // Get at the fork's overlay. A mounted fork keeps serving while we - // work: its snapshot is taken under the checkout lock, and the - // route is detached only once the fork has landed. (Detaching - // first and re-attaching on a conflict left the kernel's negative - // name cache hiding the fork on Linux FUSE, which cannot - // invalidate a route name.) - let overlay = match fork.copy_dir.as_deref() { - Some(dir) => { - let scratch = self - .handle - .scratch_checkout(fork.base) - .await - .map_err(stringify)?; - fork::capture_copy(&scratch, dir).await.map_err(stringify)?; - scratch - } - None => Arc::clone(&fork.shared), - }; + let capture_dir = fork.copy_dir.as_deref(); + let overlay = self + .fork_view(fork.base, Arc::clone(&fork.shared), capture_dir) + .await?; // A rebased fork must have resolved its markers before it can land. if let Some(conflict) = fork.conflict.as_ref() { self.refuse_unresolved_markers(&overlay, conflict).await?; diff --git a/tests/acceptance/windows-smoke.sh b/tests/acceptance/windows-smoke.sh index bc3cb1c..d34f63e 100644 --- a/tests/acceptance/windows-smoke.sh +++ b/tests/acceptance/windows-smoke.sh @@ -10,7 +10,7 @@ # (a daemon that inherits it never lets the caller see EOF) # - capture and diff under UTF-16LE names, non-ASCII included # - rewind, which renames the repo root and so trips every open handle -# - copy-mode forks and promote, the fork path Windows actually uses +# - copy forks and promote while ProjFS imports remain unproven # # Runs under Git Bash on a GitHub windows runner. ACYCLIC_BIN overrides the # binary (default: target/release/acyclic.exe). @@ -21,17 +21,36 @@ ROOT="$(cd "$HERE/../.." && pwd)" ACYCLIC="${ACYCLIC_BIN:-$ROOT/target/release/acyclic.exe}" [ -x "$ACYCLIC" ] || { echo "windows-smoke: no binary at $ACYCLIC" >&2; exit 1; } -WORK="$(mktemp -d)" +WORK="$(mktemp -d -t acyclic-windows-smoke.XXXXXXXX)" +WORK="$(cd "$WORK" && pwd -P)" REPO="$WORK/repo" cleanup() { + local status=$? + if [ "$status" -ne 0 ] && [ "${ACYCLIC_SMOKE_METRICS:-0}" = 1 ]; then + find "$WORK/stores" -name daemon.log -exec tail -n 80 {} \; 2>/dev/null || true + fi "$ACYCLIC" --repo "$REPO" stop >/dev/null 2>&1 || true - rm -rf "$WORK" 2>/dev/null || true + if [[ "$WORK" == */acyclic-windows-smoke.* && -d "$WORK" ]]; then + rm -rf -- "$WORK" 2>/dev/null || true + fi } trap cleanup EXIT fail() { echo "windows-smoke: $1" >&2; exit 1; } +metric() { + [ "${ACYCLIC_SMOKE_METRICS:-0}" = 1 ] || return 0 + local now + now="$(date +%s%N)" + echo "windows-smoke metric $1 ms=$(((now - metric_start) / 1000000))" + metric_start="$now" +} + mkdir -p "$REPO/src" +mkdir -p "$REPO/.acyclic" +# Keep the daemon's durable store inside this disposable fixture, not under +# the runner account's persistent HOME. +printf 'store_dir = "%s"\n' "$(cygpath -m "$WORK/stores")" > "$REPO/.acyclic/config.toml" cd "$REPO" git init -q . git config user.email smoke@example.com @@ -43,15 +62,18 @@ printf 'base\n' > a.txt printf 'unicode\n' > "ünïcøde.txt" git add -A git commit -qm init +metric_start="$(date +%s%N)" echo "--- init (stdout through a pipe: must not hang)" # `| cat` is the whole point: a daemon holding an inherited stdout handle # leaves this blocked forever rather than returning. "$ACYCLIC" init < /dev/null | cat > /dev/null || fail "init failed" +metric init echo "--- the daemon pipe is reachable only by this account" powershell -NoProfile -ExecutionPolicy Bypass -File "$HERE/windows-pipe-acl.ps1" \ || fail "the daemon pipe is not owner-only" +metric pipe_acl echo "--- checkpoint and timeline" # `init` pings the pipeline, whose reply is queued behind its baseline. @@ -61,11 +83,13 @@ printf 'DIFFERENT AND LONGER CONTENT\n' > src/main.rs printf 'extra\n' > added.txt "$ACYCLIC" checkpoint < /dev/null > /dev/null || fail "checkpoint failed" "$ACYCLIC" timeline < /dev/null | grep -q baseline || fail "timeline has no baseline" +metric checkpoint echo "--- diff names the changed paths" diff_out="$("$ACYCLIC" diff 1 2 < /dev/null)" grep -q 'added.txt' <<< "$diff_out" || fail "diff missed added.txt: $diff_out" grep -qi 'main.rs' <<< "$diff_out" || fail "diff missed main.rs: $diff_out" +metric diff echo "--- rewind restores content and drops added files" # `rewind` asks for confirmation on a tty; feed it the answer rather than @@ -76,20 +100,37 @@ grep -q 'ORIGINAL CONTENT LINE' src/main.rs || fail "rewind did not restore main [ ! -e added.txt ] || fail "rewind left added.txt behind" [ -e "ünïcøde.txt" ] || fail "rewind lost the non-ASCII path" [ -d .git ] || fail "rewind lost .git" +metric rewind -echo "--- forks are copies here, and promote lands them" +echo "--- copy fork writes and promote land" "$ACYCLIC" fork -n 1 < /dev/null > /dev/null || fail "fork failed" -fork_id="$("$ACYCLIC" forks < /dev/null | head -1 | awk '{print $1}')" +fork_row="$("$ACYCLIC" forks < /dev/null | head -1)" +fork_id="$(awk '{print $1}' <<< "$fork_row")" +fork_mode="$(awk '{print $2}' <<< "$fork_row")" [ -n "$fork_id" ] || fail "no fork id" +[ "$fork_mode" = copy ] || fail "Windows fork is not in verified copy mode: $fork_row" fork_dir="$(dirname "$REPO")/.$(basename "$REPO").forks/copy/$fork_id" [ -d "$fork_dir" ] || fail "no copy-fork directory at $fork_dir" +metric copy_fork printf 'FORK WORK\n' > "$fork_dir/note.txt" printf 'edited in fork\n' > "$fork_dir/a.txt" "$ACYCLIC" fork-diff "$fork_id" < /dev/null | grep -q 'note.txt' \ || fail "fork-diff did not see the fork's write" +metric fork_diff "$ACYCLIC" promote "$fork_id" < /dev/null > /dev/null || fail "promote failed" grep -q 'FORK WORK' note.txt || fail "promote did not land note.txt" grep -q 'edited in fork' a.txt || fail "promote did not land a.txt" +metric promote + +if [ "${ACYCLIC_SMOKE_METRICS:-0}" = 1 ]; then + pid_file="$(find "$WORK/stores" -name daemon.pid -print -quit)" + if [ -n "$pid_file" ]; then + ACYCLIC_SMOKE_PID="$(< "$pid_file")" powershell -NoProfile -Command \ + '$p = Get-Process -Id $env:ACYCLIC_SMOKE_PID; "windows-smoke resource cpu_ms={0} working_set_bytes={1} private_bytes={2}" -f [int64]($p.CPU * 1000), $p.WorkingSet64, $p.PrivateMemorySize64' + fi + echo "windows-smoke resource store_kib=$(du -sk "$WORK/stores" | awk '{print $1}')" + metric_start="$(date +%s%N)" +fi echo "--- daemon restart retains promoted files and timeline" "$ACYCLIC" stop < /dev/null > /dev/null || fail "stop failed" @@ -97,10 +138,12 @@ echo "--- daemon restart retains promoted files and timeline" || fail "timeline did not recover after restart" grep -q 'FORK WORK' note.txt || fail "restart lost promoted note.txt" grep -q 'edited in fork' a.txt || fail "restart lost promoted a.txt" +metric restart echo "--- a second daemon refuses the store" if "$ACYCLIC" __daemon "$REPO" < /dev/null > /dev/null 2>&1; then fail "a second daemon started against a live store" fi +metric second_daemon_refusal echo "windows-smoke: green" From 049cbf66f65a0c364106f179f94eb7cbb6f6b44f Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 01:14:15 +0100 Subject: [PATCH 022/106] Guard recursive native imports --- crates/acyclic/src/guard.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/acyclic/src/guard.rs b/crates/acyclic/src/guard.rs index be4919c..dbf60ec 100644 --- a/crates/acyclic/src/guard.rs +++ b/crates/acyclic/src/guard.rs @@ -466,6 +466,15 @@ impl MountFilesystem for GuardedMountFilesystem { self.guard(path)?; self.inner.capture_host_path(source_root, path) } + + fn capture_host_subtree( + &self, + source_root: &Path, + path: &MountPath, + ) -> Result<(), MountSourceError> { + self.guard(path)?; + self.inner.capture_host_subtree(source_root, path) + } } #[cfg(test)] From 75d8bf48b975f31c2961dd6d295c06b7d55a5b90 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 01:14:26 +0100 Subject: [PATCH 023/106] Use verified ProjFS forks when available --- crates/acyclic/src/fork.rs | 25 +++++-------------------- tests/acceptance/windows-smoke.sh | 15 +++++++++------ 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/crates/acyclic/src/fork.rs b/crates/acyclic/src/fork.rs index 94c9fe7..3f65236 100644 --- a/crates/acyclic/src/fork.rs +++ b/crates/acyclic/src/fork.rs @@ -68,9 +68,6 @@ impl MountCapability { } /// Live probe of the native mount provider. -/// -/// Windows forks use copies until `ProjFS` can capture every admitted host -/// mutation, including imports from outside its virtualized namespace. pub fn mount_capability() -> MountCapability { let probe = probe_native_mount(); let provider = match probe.kind { @@ -79,19 +76,6 @@ pub fn mount_capability() -> MountCapability { Some(NativeMountKind::WindowsProjFs) => "projfs", None => "none", }; - #[cfg(windows)] - { - let _ = probe.available; - MountCapability { - provider, - available: false, - reason: Some( - "ProjFS cannot capture every external import into a writable fork; forks use copies" - .to_owned(), - ), - } - } - #[cfg(not(windows))] MountCapability { provider, available: probe.available, @@ -120,10 +104,11 @@ pub fn mount_setup_hint() -> &'static str { ) } else if cfg!(windows) { concat!( - "forks use full copies on Windows. ProjFS cannot capture every\n", - "external import into a writable fork, so mounted forks are not\n", - "admitted. Safe Mode (dry_run) cannot shadow an existing Windows\n", - "directory with ProjFS.", + "forks will use full copies until the optional Windows Projected File System\n", + "feature is enabled and available to this process. Enable Client-ProjFS in\n", + "Windows Features to use accelerated forks. ProjFS safely rejects cross-root\n", + "directory moves that it cannot capture atomically. Safe Mode (dry_run) cannot\n", + "shadow an existing Windows directory with ProjFS.", ) } else { "native mounts are not supported on this platform; forks use full copies \ diff --git a/tests/acceptance/windows-smoke.sh b/tests/acceptance/windows-smoke.sh index d34f63e..50c0265 100644 --- a/tests/acceptance/windows-smoke.sh +++ b/tests/acceptance/windows-smoke.sh @@ -10,7 +10,7 @@ # (a daemon that inherits it never lets the caller see EOF) # - capture and diff under UTF-16LE names, non-ASCII included # - rewind, which renames the repo root and so trips every open handle -# - copy forks and promote while ProjFS imports remain unproven +# - ProjFS-accelerated forks when available, with copy fallback otherwise # # Runs under Git Bash on a GitHub windows runner. ACYCLIC_BIN overrides the # binary (default: target/release/acyclic.exe). @@ -102,16 +102,19 @@ grep -q 'ORIGINAL CONTENT LINE' src/main.rs || fail "rewind did not restore main [ -d .git ] || fail "rewind lost .git" metric rewind -echo "--- copy fork writes and promote land" +echo "--- fork writes and promote land" "$ACYCLIC" fork -n 1 < /dev/null > /dev/null || fail "fork failed" fork_row="$("$ACYCLIC" forks < /dev/null | head -1)" fork_id="$(awk '{print $1}' <<< "$fork_row")" fork_mode="$(awk '{print $2}' <<< "$fork_row")" [ -n "$fork_id" ] || fail "no fork id" -[ "$fork_mode" = copy ] || fail "Windows fork is not in verified copy mode: $fork_row" -fork_dir="$(dirname "$REPO")/.$(basename "$REPO").forks/copy/$fork_id" -[ -d "$fork_dir" ] || fail "no copy-fork directory at $fork_dir" -metric copy_fork +case "$fork_mode" in + mount) fork_dir="$(dirname "$REPO")/.$(basename "$REPO").forks/mnt/$fork_id" ;; + copy) fork_dir="$(dirname "$REPO")/.$(basename "$REPO").forks/copy/$fork_id" ;; + *) fail "Windows fork reported an unknown mode: $fork_row" ;; +esac +[ -d "$fork_dir" ] || fail "no $fork_mode fork directory at $fork_dir" +metric "${fork_mode}_fork" printf 'FORK WORK\n' > "$fork_dir/note.txt" printf 'edited in fork\n' > "$fork_dir/a.txt" "$ACYCLIC" fork-diff "$fork_id" < /dev/null | grep -q 'note.txt' \ From 1b185597ff16f2f73eac1f769630e04462fbc882 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 01:15:34 +0100 Subject: [PATCH 024/106] Fix Windows smoke metric tool resolution --- tests/acceptance/windows-smoke.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/windows-smoke.sh b/tests/acceptance/windows-smoke.sh index 50c0265..ce6c725 100644 --- a/tests/acceptance/windows-smoke.sh +++ b/tests/acceptance/windows-smoke.sh @@ -27,7 +27,7 @@ REPO="$WORK/repo" cleanup() { local status=$? if [ "$status" -ne 0 ] && [ "${ACYCLIC_SMOKE_METRICS:-0}" = 1 ]; then - find "$WORK/stores" -name daemon.log -exec tail -n 80 {} \; 2>/dev/null || true + /usr/bin/find "$WORK/stores" -name daemon.log -exec tail -n 80 {} \; 2>/dev/null || true fi "$ACYCLIC" --repo "$REPO" stop >/dev/null 2>&1 || true if [[ "$WORK" == */acyclic-windows-smoke.* && -d "$WORK" ]]; then @@ -126,7 +126,7 @@ grep -q 'edited in fork' a.txt || fail "promote did not land a.txt" metric promote if [ "${ACYCLIC_SMOKE_METRICS:-0}" = 1 ]; then - pid_file="$(find "$WORK/stores" -name daemon.pid -print -quit)" + pid_file="$(/usr/bin/find "$WORK/stores" -name daemon.pid -print -quit)" if [ -n "$pid_file" ]; then ACYCLIC_SMOKE_PID="$(< "$pid_file")" powershell -NoProfile -Command \ '$p = Get-Process -Id $env:ACYCLIC_SMOKE_PID; "windows-smoke resource cpu_ms={0} working_set_bytes={1} private_bytes={2}" -f [int64]($p.CPU * 1000), $p.WorkingSet64, $p.PrivateMemorySize64' From 6422638295bc31143a649b45d9d5e7257ebe4da5 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 01:23:11 +0100 Subject: [PATCH 025/106] Track folded SDK dependencies --- Cargo.lock | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bebc90..ae3face 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,7 +29,6 @@ dependencies = [ name = "acyclic-fs" version = "0.2.0-rc.5" dependencies = [ - "acyclic-linux-io", "acyclic-objects", "acyclic-stream", "async-trait", @@ -42,6 +41,7 @@ dependencies = [ "fuser", "futures", "hex", + "io-uring", "libc", "notify", "prost", @@ -57,19 +57,10 @@ dependencies = [ "windows", ] -[[package]] -name = "acyclic-linux-io" -version = "0.1.0-rc.1" -dependencies = [ - "io-uring", - "libc", -] - [[package]] name = "acyclic-objects" version = "1.0.0-rc.4" dependencies = [ - "acyclic-linux-io", "async-trait", "blake3", "bytes", From a7d1ed2d6ed8b0f83d667b9e19363cd13c0d35ab Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 01:40:47 +0100 Subject: [PATCH 026/106] Remove legacy Safe Mode surface --- CHANGELOG.md | 23 ++- CONTRIBUTING.md | 4 +- README.md | 7 +- SECURITY.md | 2 +- crates/acyclic/src/config.rs | 27 ++- crates/acyclic/src/fork.rs | 121 +++--------- crates/acyclic/src/guard.rs | 4 +- crates/acyclic/src/main.rs | 78 +------- crates/acyclic/src/pipeline.rs | 259 +------------------------- crates/acyclic/src/proto.rs | 32 +--- crates/acyclic/src/server.rs | 211 +-------------------- crates/acyclic/tests/fork.rs | 71 +------- docs/design/00-overview.md | 6 +- docs/design/01-rewind.md | 2 +- docs/design/03-forks.md | 2 +- docs/design/04-safe-mode.md | 122 ------------- docs/design/05-monorepo.md | 2 +- docs/design/06-installation.md | 2 +- docs/design/07-compliance.md | 7 +- docs/design/implementation-forks.md | 2 +- docs/design/implementation-merge.md | 25 +-- docs/design/implementation-rewind.md | 4 +- docs/design/spec-forks.md | 7 +- docs/windows-verification.md | 12 +- tests/acceptance/run-all.sh | 2 +- tests/acceptance/safe-mode.sh | 263 --------------------------- 26 files changed, 117 insertions(+), 1180 deletions(-) delete mode 100644 docs/design/04-safe-mode.md delete mode 100755 tests/acceptance/safe-mode.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 27ad679..ae56ea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ Pre-1.0; `main` is the only supported line (see `SECURITY.md`). ### Changed +- **The unfinished session-shadowing feature was removed.** It depended on + replacing the live repository with a native mount, was not wired into host + approval flows, and could not work consistently across platforms. Existing + `dry_run` configuration is now rejected; explicit forks remain available. + Before replacing the old binary, use that binary to stop every daemon with + an active shadow mount. A new protocol-v2 CLI cannot stop a protocol-v1 + daemon. If the old daemon already crashed, recover the real tree with + `fusermount3 -u ` (or `fusermount -u `) on Linux, or + `umount -f ` (then `diskutil unmount force ` if needed) on macOS. - **A cold daemon no longer delays the agent's first turn.** The session-start hook waits at most 300ms for the daemon; past that it prints a one-line notice and returns while the first snapshot builds in the background (251s @@ -19,8 +28,8 @@ Pre-1.0; `main` is the only supported line (see `SECURITY.md`). O(tree) (7s per fork and 6s per promote on 5,000 files); it now runs on the existing idle and every-N timers, like every other checkpoint. A fork is cut at its exact base generation, published or not. -- **The daemon exits after an hour idle** with no session, fork, or Safe Mode - session (`daemon_idle_exit_ms`, 0 disables). Twenty daemons were found alive +- **The daemon exits after an hour idle** with no session or fork + (`daemon_idle_exit_ms`, 0 disables). Twenty daemons were found alive on one machine, eleven for repos that no longer existed. - `acyclic status` reports the store size from a cache refreshed in the background instead of walking the object directory on every call. @@ -67,8 +76,6 @@ were 0.0.1. Host coverage, Speculation and Windows are what this release adds. the conversation, not just by commit. - **Forks** (Launch 3) — copy-on-write materialization for running parallel attempts, with promotion back via three-way merge. -- **Safe Mode** (Launch 4) — session redirection and interposition so agent mistakes land in a - redirected session rather than the working tree; needs the native mount layer. - **Speculation** — the daemon computes what the agent is about to ask for while nobody is waiting: the previous-session brief when a session ends, and (optionally, with a model command the developer names) a summary of each turn at the turn boundary. Results are keyed @@ -79,10 +86,10 @@ were 0.0.1. Host coverage, Speculation and Windows are what this release adds. - Published to npm as `@acyclic-labs/plugin`. - **Windows x64 support** — the daemon transport gains a named-pipe implementation alongside the Unix domain socket, and host names are encoded per platform (UTF-16LE on Windows) so capture, - diff, exclusions and the Safe Mode guard agree with the filesystem. Verified on Windows 11 and - covered by a `windows-2022` CI job. Two caveats: forks are always full copies there (ProjFS - projects a fork but does not carry writes back, so a mounted fork would silently lose work) and - Safe Mode needs a real mount, so it is unavailable. See `docs/windows-verification.md`. + diff, exclusions and guarded fork paths agree with the filesystem. Verified on Windows 11 and + covered by a `windows-2022` CI job. Forks use full copies there because ProjFS projects a fork + but does not carry writes back, so a mounted fork would silently lose work. See + `docs/windows-verification.md`. ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c8d1a2..1d4f9e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ Acceptance suites (end-to-end, run against a real repo) live in `tests/acceptanc tests/acceptance/run-all.sh ``` -Individual suites (`journey.sh`, `timeline.sh`, `forks.sh`, `merge.sh`, `safe-mode.sh`, etc.) can +Individual suites (`journey.sh`, `timeline.sh`, `forks.sh`, `merge.sh`, etc.) can be run with `bash tests/acceptance/.sh` if you're iterating on one feature. The `*-e2e.sh` suites drive the real host CLIs (Claude Code, Codex, cursor-agent, OpenCode) and @@ -106,7 +106,7 @@ exempt from this check. ## Design context `docs/design/` has the design docs and implementation notes behind the bigger features (Rewind, -Timeline, Forks, Safe Mode). Worth a skim before working on any of them — they capture the +Timeline, Forks). Worth a skim before working on any of them — they capture the tradeoffs and constraints that shaped the current architecture, including a few (like snapshot GC) that are intentionally deferred. diff --git a/README.md b/README.md index 3d79466..6d0d687 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Checkpoint every agent action, rewind exactly, see the blast radius. The store c **A local product with plugin distribution.** The product is an agent-native state engine that runs on your machine — snapshots, forks, and indexing over your working tree. The plugins are thin adapters that deliver it through Claude Code, Codex, OpenCode, any agent that can run a shell command, and Claude Desktop over MCP. The engine is the moat; the plugins are the channel. -> Status: Launches 1–4 built (Rewind, Timeline, Forks, Safe Mode), acceptance suites green on macOS and Linux, published to npm as `@acyclic-labs/plugin`. Launch 1's release gate is met: snapshot exclusions, a store-growth proof, license scanning, an attested SBOM per binary, `scripts/install.sh`, and a clean-machine install test. What v1 deliberately does not do is prune or purge history; see [Retention and purge](#retention-and-purge). Launch 5 (Monorepo) is spec. The spec lives on the [Acyclic plugins docs page](https://acyclic.dev/docs/plugins). +> Status: Launches 1–3 built (Rewind, Timeline, Forks), acceptance suites green on macOS, Linux, and Windows, published to npm as `@acyclic-labs/plugin`. Launch 1's release gate is met: snapshot exclusions, a store-growth proof, license scanning, an attested SBOM per binary, `scripts/install.sh`, and a clean-machine install test. What v1 deliberately does not do is prune or purge history; see [Retention and purge](#retention-and-purge). Launch 5 (Monorepo) is spec. The spec lives on the [Acyclic plugins docs page](https://acyclic.dev/docs/plugins). ## Install @@ -67,7 +67,7 @@ Not yet covered: an `install` writer for Codex's MCP config (TOML), Kimi Code CL | `commit_every` / `commit_idle_ms` | `25` / `60000` | How often per-tool-call checkpoints are published to the durable store. | | `auto_checkpoint_idle_ms` | `5000` | Idle-timer safety net: checkpoints changes on its own once the watcher has been quiet this long, for hosts with no lifecycle-hook API (Claude Desktop). `0` disables it. Cheap no-op for hooked hosts, which already drain the watcher themselves. | | `quiesce_ms` / `quiesce_cap_ms` | `50` / `500` | Watcher quiet window before a capture. | -| `dry_run` / `guarded_paths` | `false` / `[]` | Safe Mode (Launch 4). | +| `guarded_paths` | `[]` | Repo-relative prefixes that mounted forks cannot write. Copy-mode forks do not enforce this mount-layer policy. | | `[decompose]` / `[merge]` | | Fork decomposition policy and merge limits (Launch 3). | | `store_dir` | `~/.local/share/acyclic/stores` | Where stores live. Never inside the repo. | @@ -108,7 +108,7 @@ Median lead is the number to watch: it is how far ahead of the request a claimed `acyclic status` reports the store size; trash is pruned by TTL. The store itself is never garbage-collected in v1, on purpose. At the pinned `acyclic-fs` revision a generation stays reachable only while it is a workspace head or carries a retention fact (checkpoint label, pin, fork base), retention facts cannot be released, and closure proofs do not follow generation parents. So the fs collector would either destroy every checkpoint but the head or, if every checkpoint were pinned first, never free anything again. Purge-through-history has the same dependency: content cannot be physically removed from a retained generation. Both land when the fs grows a retention-release fact; until then, keep secrets out of the store with `exclude`, which is the compliance control that ships. Details and the upstream ask are in `docs/design/implementation-rewind.md`, Phase 4. -Known caveats: mtimes are not restored on rewind, a rewind warrants an editor reload, forks and Safe Mode sessions do not see excluded paths, and baseline capture runs at roughly 230 s/GiB on first `init`. +Known caveats: mtimes are not restored on rewind, a rewind warrants an editor reload, forks do not see excluded paths, and baseline capture runs at roughly 230 s/GiB on first `init`. ## Thesis @@ -130,7 +130,6 @@ One engine, thin adapters: | 1 | Rewind | Merkle snapshot store + host hooks | Never fear letting the agent loose | built (`tests/acceptance/journey.sh`, `crash.sh`, `soak.sh`, `latency.sh`, `claude-e2e.sh`) | | 2 | Timeline | Turn-linked metadata index | The repo at any point in the conversation | built (`timeline.sh`) | | 3 | Forks | Copy-on-write materialization | N parallel attempts, pick the winner | built: mounted forks, promote with three-way merge (`forks.sh`, `merge.sh`, `claude-merge-e2e.sh`) | -| 4 | Safe Mode | Session redirection + interposition | Agents on the codebase, not agents' mistakes in it | built, needs the native mount layer (`safe-mode.sh`) | | 5 | Monorepo | Merkle-aware content + symbol index | The repo that finally works with agents | not started | Run everything with `tests/acceptance/run-all.sh`; the live Claude Code scenarios are gated by `ACYCLIC_E2E=1`. diff --git a/SECURITY.md b/SECURITY.md index 326903c..c4a3500 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,7 @@ Please do not open public issues for security vulnerabilities. Email **security@ The engine (`acyclic` CLI and daemon), the host-tool adapters (Claude Code, Codex, OpenCode), and the release pipeline (signed artifacts, SBOM, provenance). -Of particular interest: snapshot-store data exposure (secrets retained in checkpoints), interposition bypasses (guarded paths, dry-run escapes), and supply-chain issues in the adapters. +Of particular interest: snapshot-store data exposure (secrets retained in checkpoints), guarded fork interposition bypasses, and supply-chain issues in the adapters. ## Supported versions diff --git a/crates/acyclic/src/config.rs b/crates/acyclic/src/config.rs index b49fb43..0effa18 100644 --- a/crates/acyclic/src/config.rs +++ b/crates/acyclic/src/config.rs @@ -28,18 +28,15 @@ pub struct Config { /// records nothing. Zero disables it. pub auto_checkpoint_idle_ms: u64, /// The daemon exits after this long (ms) with no request, no live - /// session, no live fork and no Safe Mode session. It restarts on the + /// session and no live fork. It restarts on the /// next session start. Zero keeps it alive forever. pub daemon_idle_exit_ms: u64, /// Days a rewound-away tree is kept in the store's trash. pub trash_ttl_days: u32, /// Override for the store directory (defaults to the per-machine root). pub store_dir: Option, - /// Safe Mode: root every session in a fork by default, gated on an - /// approved diff before anything reaches the real tree. - pub dry_run: bool, - /// Safe Mode: path prefixes (relative to the repo root) no fork or - /// scratch tree may write to, enforced at the native mount layer. + /// Path prefixes (relative to the repo root) that mounted forks may not + /// write to, enforced at the native mount layer. pub guarded_paths: Vec, /// Snapshot exclusions: repo-relative paths (a file, or a directory and /// everything under it) that never enter a checkpoint. For secrets and @@ -123,7 +120,6 @@ impl Default for Config { daemon_idle_exit_ms: 3_600_000, trash_ttl_days: 7, store_dir: None, - dry_run: false, guarded_paths: Vec::new(), exclude: Vec::new(), decompose: Decompose::default(), @@ -233,16 +229,15 @@ mod tests { } #[test] - fn safe_mode_fields_parse_from_repo_config() { + fn guarded_paths_parse_from_repo_config() { let repo = tempfile::tempdir().expect("tempdir"); std::fs::create_dir(repo.path().join(crate::product::repo_config_dir())).expect("dir"); std::fs::write( repo.path().join(crate::product::repo_config_file()), - "dry_run = true\nguarded_paths = [\".env\", \"migrations/\"]\n", + "guarded_paths = [\".env\", \"migrations/\"]\n", ) .expect("write"); let config = Config::load_layered(None, repo.path()).expect("load"); - assert!(config.dry_run); assert_eq!(config.guarded_paths, vec![".env", "migrations/"]); } @@ -303,6 +298,18 @@ mod tests { assert!(Config::load_layered(None, repo.path()).is_err()); } + #[test] + fn removed_dry_run_key_is_rejected() { + let repo = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir(repo.path().join(crate::product::repo_config_dir())).expect("dir"); + std::fs::write( + repo.path().join(crate::product::repo_config_file()), + "dry_run = true\n", + ) + .expect("write"); + assert!(Config::load_layered(None, repo.path()).is_err()); + } + /// The layering the doc comment and README promise: a key set only in /// the machine config survives a repo config that does not mention it. /// Before the per-key merge, parsing the repo file discarded the whole diff --git a/crates/acyclic/src/fork.rs b/crates/acyclic/src/fork.rs index 3f65236..ff79275 100644 --- a/crates/acyclic/src/fork.rs +++ b/crates/acyclic/src/fork.rs @@ -92,27 +92,23 @@ pub fn mount_setup_hint() -> &'static str { "forks will use full copies until /dev/fuse is usable. To enable mounts:\n", " sudo modprobe fuse # load the kernel module\n", " sudo usermod -aG fuse \"$USER\" # if /dev/fuse is group-restricted; log in again\n", - " docker run --device /dev/fuse --cap-add SYS_ADMIN ... # inside a container\n", - "Safe Mode (dry_run) needs mounts and refuses to start without them.", + " docker run --device /dev/fuse --cap-add SYS_ADMIN ... # inside a container", ) } else if cfg!(target_os = "macos") { concat!( "forks will use full copies: the built-in NFS mount tools (/sbin/mount_nfs, /sbin/umount)\n", "are missing or blocked by policy. No extra software is needed on macOS;\n", - "ask your administrator to allow loopback NFS mounts.\n", - "Safe Mode (dry_run) needs mounts and refuses to start without them.", + "ask your administrator to allow loopback NFS mounts.", ) } else if cfg!(windows) { concat!( "forks will use full copies until the optional Windows Projected File System\n", "feature is enabled and available to this process. Enable Client-ProjFS in\n", "Windows Features to use accelerated forks. ProjFS safely rejects cross-root\n", - "directory moves that it cannot capture atomically. Safe Mode (dry_run) cannot\n", - "shadow an existing Windows directory with ProjFS.", + "directory moves that it cannot capture atomically.", ) } else { - "native mounts are not supported on this platform; forks use full copies \ - and Safe Mode (dry_run) is unavailable." + "native mounts are not supported on this platform; forks use full copies." } } @@ -122,7 +118,7 @@ pub fn forks_copy_root(repo_root: &Path) -> Option { } /// Captures the current content of `root` into a fork's overlay, so that -/// promote/resolve see it exactly as they would see writes through a mount. +/// promote sees it exactly as it would see writes through a mount. /// The overlay must be pristine (a fresh fork seed): capture is a full-tree /// baseline against the checkout, so only paths that actually differ from /// the base become pending mutations. @@ -168,21 +164,6 @@ pub enum PromoteOutcome { Conflict { message: String }, } -/// Result of the commit half of a Safe Mode session resolve (see -/// [`crate::pipeline::PipelineHandle::resolve_session`]) — the swap itself -/// is deferred to a separate `apply_session` call so the caller can show an -/// approval-gated diff in between. -#[derive(Clone, Debug)] -pub enum SessionResolveOutcome { - /// The overlay committed cleanly; `generation` is ready for - /// `apply_session`, or can simply be left unswapped (Safe Mode reject). - Resolved { generation: GenerationId }, - /// Nothing was written; there is nothing to diff or apply. - NoChanges, - /// The mainline moved past the fork's base — same v1 stance as promote. - Conflict { message: String }, -} - /// Where a repo's fork workspaces live: a sibling of the repo, outside the /// working tree so capture never sees them. pub fn forks_root(repo_root: &Path) -> Option { @@ -235,87 +216,41 @@ pub fn sweep_stale_forks(repo_root: &Path) { let _ = std::fs::remove_dir(&root); } -/// Best-effort cleanup of a Safe Mode shadow mount a dead daemon left -/// directly on `repo_root` itself. Unlike [`sweep_stale_forks`], the -/// directory is never removed here -- it IS the real repo -- only -/// force-unmounted so its real content reappears. A no-op if nothing is -/// mounted there. -pub fn sweep_stale_dry_session(repo_root: &Path) { - #[cfg(target_os = "macos")] - { - let _ = std::process::Command::new("umount") - .arg("-f") - .arg(repo_root) - .status(); - } - #[cfg(target_os = "linux")] - { - let _ = std::process::Command::new("fusermount") - .arg("-u") - .arg(repo_root) - .status(); - } - // A ProjFS virtualization root stops with the process that owned it, so - // a dead daemon leaves nothing mounted over the repo to reap. - #[cfg(windows)] - { - let _ = repo_root; - } -} - -/// Reaps a Safe Mode shadow left by a *crashed* daemon before the caller -/// touches `repo`. A dead daemon's shadow is a loopback-NFS/FUSE mountpoint -/// whose server is gone, so every `stat`/`open` under it (config load, path -/// canonicalization) blocks indefinitely — the CLI would hang before it -/// could even spawn a fresh daemon to clean up. -/// -/// Distinguishing a dead shadow from a live session (whose shadow is fine) -/// without a store/config lookup — which would itself stat `repo` — is done -/// by probing: a live server answers a `stat` immediately, while a dead one -/// either blocks or fails (the NFS layer surfaces `ETIMEDOUT`/`ENOTCONN`). -/// So this force-unmounts whenever a bounded probe does not cleanly succeed; -/// a healthy repo always stats OK and is never disturbed, and a `umount` of a -/// path that is not actually a mount is a harmless no-op. -pub fn reap_dead_shadow(repo: &Path) { +/// Recovers a repository still covered by a shadow mount from a legacy +/// `dry_run` daemon. New versions no longer create these mounts, but upgrade +/// must remain able to expose the real repository before opening it. +pub fn reap_legacy_shadow(repo: &Path) { #[cfg(any(target_os = "macos", target_os = "linux"))] { use std::time::Duration; - // Resolve to an absolute mountpoint via the (always-live) parent, so - // the probe/unmount target is stable without stat-ing `repo` itself. let target = match (repo.parent(), repo.file_name()) { - (Some(parent), Some(name)) => match parent.canonicalize() { - Ok(parent) => parent.join(name), - Err(_) => repo.to_path_buf(), - }, + (Some(parent), Some(name)) => parent + .canonicalize() + .map_or_else(|_| repo.to_path_buf(), |parent| parent.join(name)), _ => repo.to_path_buf(), }; let probe = target.clone(); - let (tx, rx) = std::sync::mpsc::channel(); - // Detached: if the stat is truly wedged it never returns, but the - // force-unmount below releases it and this short-lived CLI exits. - std::thread::spawn(move || { - let _ = tx.send(std::fs::metadata(&probe).is_ok()); - }); - let healthy = matches!(rx.recv_timeout(Duration::from_secs(5)), Ok(true)); - if !healthy { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || drop(sender.send(std::fs::metadata(probe).is_ok()))); + if !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok(true)) { #[cfg(target_os = "macos")] - let _ = std::process::Command::new("umount") - .arg("-f") - .arg(&target) - .status(); + drop( + std::process::Command::new("umount") + .arg("-f") + .arg(target) + .status(), + ); #[cfg(target_os = "linux")] - let _ = std::process::Command::new("fusermount") - .arg("-u") - .arg(&target) - .status(); + drop( + std::process::Command::new("fusermount") + .arg("-u") + .arg(target) + .status(), + ); } } - // A ProjFS shadow dies with the daemon that projected it, so there is no - // wedged mountpoint to probe for and nothing to force-unmount. #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - let _ = repo; - } + let _ = repo; } #[cfg(test)] diff --git a/crates/acyclic/src/guard.rs b/crates/acyclic/src/guard.rs index dbf60ec..c2c95c7 100644 --- a/crates/acyclic/src/guard.rs +++ b/crates/acyclic/src/guard.rs @@ -5,7 +5,7 @@ //! interposition, and its routing is single-level and non-overlaying — it //! cannot layer a read-only view over part of a writable fork. Guarding //! therefore wraps the *inner* source (a fork's `CheckoutMountSource`) -//! directly, before it is ever registered as a route or shadow mount, so +//! directly, before it is registered as a fork route, so //! guarded prefixes work at any depth. use acyclic_fs::kernel::FileMetadata; @@ -99,7 +99,7 @@ impl GuardedMountFilesystem { /// Whether a mutating call to `path` must be refused: either it falls /// under a configured guarded prefix, or its leaf is a macOS `AppleDouble` /// sidecar (`._X`). Sidecars are written by the macOS client over the - /// mount to carry a file's xattrs / resource fork; in a Safe Mode or fork + /// mount to carry a file's xattrs / resource fork; in a fork /// projection they are pure transport noise that would otherwise pollute /// the session diff and litter the real tree on apply, and a guarded /// file's metadata must not leak into one either. Dropping every sidecar diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index fa7bb69..6a46099 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -216,17 +216,6 @@ enum Command { /// Record a host session ending (hook use). #[command(hide = true)] SessionEnd { session_id: String }, - /// Safe Mode: commit a session's shadow fork and show what it would - /// change, without touching the real tree yet. - #[command(hide = true)] - SessionResolve { session_id: String }, - /// Safe Mode: apply a `session-resolve`d session's changes to the real - /// tree. - #[command(hide = true)] - SessionApply { session_id: String }, - /// Safe Mode: discard a `session-resolve`d session without applying it. - #[command(hide = true)] - SessionDiscard { session_id: String }, /// Internal: the per-repo daemon process. #[command(name = "__daemon", hide = true)] Daemon { repo_root: PathBuf }, @@ -251,12 +240,7 @@ fn main() { } } let repo_arg = cli.repo.clone().unwrap_or_else(|| PathBuf::from(".")); - // Before touching the repo path (canonicalize, config load, socket): a - // crashed Safe Mode daemon can leave a dead shadow mount over the repo - // root that wedges every stat under it. Force-unmount it first (a no-op - // unless a bounded probe shows the mount is genuinely wedged), so the - // real tree is back before we read anything. - acyclic::fork::reap_dead_shadow(&repo_arg); + acyclic::fork::reap_legacy_shadow(&repo_arg); let repo_arg = acyclic::rewind::recover_before_repo_open(&repo_arg).unwrap_or_else(|error| { eprintln!("{}: rewind recovery: {error}", product::NAME); std::process::exit(1); @@ -396,15 +380,12 @@ fn connect(repo: &Path, spawn: Spawn) -> Result { Client::connect(&paths.socket(), repo, &log, spawn) } -/// One line on what forks and Safe Mode can do here, plus setup steps when -/// the host lacks a mount provider. Shown by `init` and `install`. +/// One line on native fork support, plus setup steps when the host lacks a +/// mount provider. Shown by `init` and `install`. fn print_mount_capability() { let capability = acyclic::fork::mount_capability(); if capability.available { - println!( - "mounts: {} (forks and Safe Mode available)", - capability.provider - ); + println!("mounts: {} (forks mount)", capability.provider); } else { println!( "mounts: unavailable ({})", @@ -992,13 +973,10 @@ fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), Str println!("unpublished: {}", info.unpublished); println!("store size: {}", human_bytes(info.store_bytes)); if info.mount_available { - println!( - "mounts: {} (forks mount, Safe Mode on)", - info.mount_provider - ); + println!("mounts: {} (forks mount)", info.mount_provider); } else { println!( - "mounts: unavailable ({}) — forks copy, Safe Mode off", + "mounts: unavailable ({}) — forks copy", info.mount_reason.as_deref().unwrap_or("unknown reason") ); } @@ -1052,50 +1030,6 @@ fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), Str client.call(proto::Op::SessionEnd { session_id })?; Ok(()) } - Command::SessionResolve { session_id } => { - let reply = client.call(proto::Op::SessionResolve { session_id })?; - let proto::Reply::SessionPending(info) = reply else { - return Err("unexpected reply".into()); - }; - if info.diff.is_empty() { - println!("session {}: no changes", info.session_id); - return Ok(()); - } - for entry in &info.diff { - println!("{} {}", entry.change.tag(), entry.path); - } - println!( - "{} paths changed; run `{NAME} session-apply {}` to land them or \ - `{NAME} session-discard {}` to throw them away", - info.diff.len(), - info.session_id, - info.session_id - ); - Ok(()) - } - Command::SessionApply { session_id } => { - step_aside(); - let reply = client.call(proto::Op::SessionApply { session_id })?; - let proto::Reply::Promote(info) = reply else { - return Err("unexpected reply".into()); - }; - match info.old_tree { - Some(old_tree) => { - println!( - "applied: working tree now at {}", - short_hex(&info.generation) - ); - println!("old tree kept at {old_tree}"); - println!("note: {}", info.warning); - } - None => println!("session had no changes; nothing to land"), - } - Ok(()) - } - Command::SessionDiscard { session_id } => { - client.call(proto::Op::SessionDiscard { session_id })?; - Ok(()) - } Command::Init | Command::Policy | Command::Daemon { .. } diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index b61fb80..d954904 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use crate::config::Config; use crate::diff::{self, FileChange}; use crate::exclude::Exclusions; -use crate::fork::{ForkSeed, PromoteOutcome, SessionResolveOutcome, SharedLocalCheckout}; +use crate::fork::{ForkSeed, PromoteOutcome, SharedLocalCheckout}; use crate::index::{Attribution, CheckpointKind, CheckpointRow, Index}; use crate::rewind::{self, RestoreOutcome, RewindOutcome}; use crate::store::Store; @@ -219,25 +219,9 @@ enum Request { label: String, reply: oneshot::Sender>, }, - ResolveSession { - shared: Arc, - base: GenerationId, - label: String, - reply: oneshot::Sender>, - }, - ApplySession { - generation: GenerationId, - base: GenerationId, - label: String, - reply: oneshot::Sender>, - }, Status { reply: oneshot::Sender>, }, - SetShadowed { - active: bool, - reply: oneshot::Sender>, - }, SessionStarted { session_id: String, host: String, @@ -587,56 +571,10 @@ impl PipelineHandle { )? } - /// Safe Mode's commit half of promote: commits the overlay (or reports - /// a conflict) without touching the real tree, so the caller can show - /// an approval-gated diff before deciding whether to `apply_session`. - pub async fn resolve_session( - &self, - shared: Arc, - base: GenerationId, - label: String, - ) -> Result { - request!( - self, - ResolveSession { - shared: shared, - base: base, - label: label - } - )? - } - - /// Safe Mode's swap half of promote: lands an already-`resolve_session`d - /// generation onto the real tree. - pub async fn apply_session( - &self, - generation: GenerationId, - base: GenerationId, - label: String, - ) -> Result { - request!( - self, - ApplySession { - generation: generation, - base: base, - label: label - } - )? - } - pub async fn status(&self) -> Result { request!(self, Status {})? } - /// Safe Mode: while a session's fork is shadow-mounted over the real repo - /// root, mainline capture must pause — the pipeline's watcher would - /// otherwise observe the fork's content and the mount/unmount lifecycle - /// (which can map to the volume root) instead of real-tree mutations. - /// `resolve_session`/`apply_session` clear it and rebuild the watcher. - pub async fn set_shadowed(&self, active: bool) -> Result<()> { - request!(self, SetShadowed { active: active })? - } - pub async fn session_started(&self, session_id: String, host: String) -> Result<()> { request!( self, @@ -679,7 +617,7 @@ enum RootHint { /// Removes hints that target the volume root from a batch. The engine /// refuses any mutation of the root ("mutation cannot target the volume /// root"), and such hints only ever come from mount lifecycle events on the -/// repo directory (Safe Mode shadowing it, or a fork session tearing down), +/// repo directory (for example, a fork session tearing down), /// never from the user's edits. fn strip_root_hints(batch: WatchBatch) -> (WatchBatch, RootHint) { let WatchBatch::Changes { @@ -778,9 +716,6 @@ struct Pipeline { last_checkpoint_row: Option, checkpoints_since_commit: u32, last_activity: Instant, - /// A Safe Mode session's fork is shadow-mounted over the repo root: - /// mainline capture is suspended until `resolve_session`/`apply_session`. - shadowed: bool, /// Watcher invalidations and recovery rescans since start; see /// [`WatcherHealth`]. watcher_health: WatcherHealth, @@ -868,12 +803,9 @@ fn fail_request(request: Request, message: &str) { Request::MaterializePaths { reply, .. } => drop(reply.send(Err(error()))), Request::RecordGeneration { reply, .. } => drop(reply.send(Err(error()))), Request::SnapshotOverlay { reply, .. } => drop(reply.send(Err(error()))), - Request::ResolveSession { reply, .. } => drop(reply.send(Err(error()))), - Request::ApplySession { reply, .. } => drop(reply.send(Err(error()))), Request::RestorePath { reply, .. } => drop(reply.send(Err(error()))), Request::RestorePaths { reply, .. } => drop(reply.send(Err(error()))), Request::TurnStarted { reply, .. } => drop(reply.send(Err(error()))), - Request::SetShadowed { reply, .. } => drop(reply.send(Err(error()))), Request::Status { reply } => drop(reply.send(Err(error()))), Request::SessionStarted { reply, .. } | Request::SessionEnded { reply, .. } => { drop(reply.send(Err(error()))); @@ -922,7 +854,6 @@ impl Pipeline { last_checkpoint_row: None, checkpoints_since_commit: 0, last_activity: Instant::now(), - shadowed: false, watcher_health: WatcherHealth::default(), auto_pending: None, }; @@ -1286,24 +1217,6 @@ impl Pipeline { let _ = reply.send(self.promote(shared, base, &label).await); false } - Request::ResolveSession { - shared, - base, - label, - reply, - } => { - let _ = reply.send(self.resolve_session(shared, base, &label).await); - false - } - Request::ApplySession { - generation, - base, - label, - reply, - } => { - let _ = reply.send(self.apply_session(generation, base, &label).await); - false - } Request::Status { reply } => { let _ = reply.send(Ok(StatusReport { state: self.state, @@ -1314,11 +1227,6 @@ impl Pipeline { })); false } - Request::SetShadowed { active, reply } => { - self.shadowed = active; - let _ = reply.send(Ok(())); - false - } Request::SessionStarted { session_id, host, @@ -1357,26 +1265,6 @@ impl Pipeline { attribution: &Attribution, ) -> Result { let started = Instant::now(); - if self.shadowed { - // A Safe Mode session's fork is mounted over the repo root; the - // real tree is frozen and the watcher sees only fork/mount noise. - // Record a noop so the hook gets a clean reply, but never drain - // the watcher or snapshot the mainline mid-session. - crate::trace!( - "pipeline", - "checkpoint kind={:?}: shadowed -> noop row, watcher untouched", - kind - ); - let row = self - .index - .record(self.last_generation, CheckpointKind::Noop, attribution)?; - self.last_checkpoint_row = Some(row); - return Ok(CheckpointOutcome { - row_id: row, - generation: self.last_generation, - kind: CheckpointKind::Noop, - }); - } if self.state != State::Ready { // A failed recovery leaves state at Baselining; a request is the // natural moment to retry rather than staying down forever. @@ -1639,7 +1527,7 @@ impl Pipeline { /// meantime. Never records a row when nothing changed, so it cannot spam /// the timeline. async fn auto_checkpoint(&mut self) { - if self.config.auto_checkpoint_idle_ms == 0 || self.shadowed || self.state != State::Ready { + if self.config.auto_checkpoint_idle_ms == 0 || self.state != State::Ready { return; } // Like `idle_commit`, failures here are advisory: the pending state @@ -1837,11 +1725,6 @@ impl Pipeline { safety: bool, label: Option, ) -> Result { - if self.shadowed { - return Err(EngineError::Restore( - "a Safe Mode session is shadowing the repo root; resolve it first".into(), - )); - } for path in paths { if self.exclusions.covers_host(path) { return Err(EngineError::Restore(format!( @@ -2051,7 +1934,7 @@ impl Pipeline { Ok(ForkSeed { // Native close/flush must never publish: sibling forks share one // volume head, so a seal on close makes the next fork's mutations - // Stale. Promote/resolve are the only commits. + // stale. Promote is the only commit. shared: Arc::new(SharedLocalCheckout::with_publication( checkout, MountPublication::Manual, @@ -2167,140 +2050,6 @@ impl Pipeline { }) } - /// The commit half of promote, split out for Safe Mode: publishes the - /// mainline safety net and commits the fork's overlay, but never - /// touches the real tree — the caller diffs `base` against the - /// returned generation and decides whether to `apply_session` it. - async fn resolve_session( - &mut self, - shared: Arc, - base: GenerationId, - label: &str, - ) -> Result { - // The server unmounts the shadow before calling us, so the pipeline - // watcher's queue is full of mount-teardown hints — some of which map - // to the volume root and would fail capture outright. Leave shadow - // mode and swap in a fresh watcher on the now-real tree to DISCARD - // that queue. The pipeline's own checkout never moved during the - // session (mainline checkpoints no-op while shadowed), so it still - // sits at `base`; we must not re-baseline/publish it here or the - // mainline HEAD would advance past the fork and the overlay commit - // below would spuriously conflict. - if self.shadowed { - self.shadowed = false; - self.reset_watch().await?; - let _ = self - .watch - .as_mut() - .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? - .finish_rescan() - .map_err(EngineError::fs("finish rescan"))?; - self.state = State::Ready; - } else { - // No shadow (direct callers, e.g. tests): the watcher is trusted, - // so fold any pending real-tree change into the safety net. - self.drain_watcher().await?; - } - let safety = self.checkpoint_engine().await?; - let safety_row = self.index.record( - safety, - CheckpointKind::PreRewind, - &Attribution { - label: Some(format!("before {label}")), - ..Attribution::default() - }, - )?; - self.last_generation = safety; - self.last_checkpoint_row = Some(safety_row); - // The safety checkpoint stays UNPUBLISHED (checkpoint, not commit): - // publishing it would move the authority head off `base`, and Safe - // Mode must leave the head at base until `apply_session` so that - // `session-discard` truly changes nothing and the next session forks - // cleanly. Unpublished checkpoint generations are fully restorable. - let _ = base; // conflict against a moved mainline is detected at apply - - // Snapshot the fork's overlay as an unpublished generation: diffable - // and rewind-applyable by id, without advancing the head. - let generation = { - let guard = shared.lock().await; - if !guard.has_pending_mutations() { - return Ok(SessionResolveOutcome::NoChanges); - } - guard - .checkpoint(WorkCounters::UNBOUNDED, &self.cancel) - .await - .map_err(EngineError::fs("session checkpoint"))? - .value - }; - Ok(SessionResolveOutcome::Resolved { generation }) - } - - /// The swap half of promote, split out for Safe Mode: lands an - /// already-committed generation (from `resolve_session`) onto the real - /// tree, exactly like `promote`'s own tail. - async fn apply_session( - &mut self, - generation: GenerationId, - base: GenerationId, - label: &str, - ) -> Result { - self.state = State::Rewinding; - self.drain_watcher().await?; - if self.checkpoint_engine().await? != base { - self.state = State::Ready; - return Ok(PromoteOutcome::Conflict { - message: format!( - "the working tree moved past the session's base ({}); Safe Mode in v1 requires an unmoved mainline", - crate::generation_hex(base) - ), - }); - } - self.commit_engine().await?; - self.store.checkout = self - .store - .volume - .checkout( - acyclic_fs::model::GenerationSelector::Head, - crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, - ) - .await - .map_err(EngineError::fs("refresh checkout"))? - .value; - // Same v1 stance as promote: if the mainline published anything since - // the fork's base, the head has moved off `base` and applying would - // clobber it. Detected here rather than at resolve, because resolve - // deliberately leaves the head at base (unpublished overlay). - if self.store.checkout.generation_id() != base { - self.state = State::Ready; - return Ok(PromoteOutcome::Conflict { - message: format!( - "the working tree moved past the session's base ({}); \ - Safe Mode in v1 requires an unmoved mainline — rewind to \ - the base or start a new session", - crate::generation_hex(base) - ), - }); - } - let row = self.index.record( - generation, - CheckpointKind::Manual, - &Attribution { - label: Some(label.to_owned()), - ..Attribution::default() - }, - )?; - self.last_generation = generation; - self.last_checkpoint_row = Some(row); - - let swap = self.swap_root_and_rebaseline(generation).await?; - Ok(PromoteOutcome::Promoted { - generation, - old_tree: Some(swap.old_tree), - }) - } - /// Stops the old event source before the intentional root exchange. An /// old callback must never publish a hint into the new watcher's epoch. async fn swap_root_and_rebaseline( diff --git a/crates/acyclic/src/proto.rs b/crates/acyclic/src/proto.rs index 4937305..8375ce8 100644 --- a/crates/acyclic/src/proto.rs +++ b/crates/acyclic/src/proto.rs @@ -62,7 +62,7 @@ mod domain_wire_tests { } } -pub const PROTOCOL_VERSION: u32 = 1; +pub const PROTOCOL_VERSION: u32 = 2; /// The kinds a client may ask for. Bookkeeping kinds (`baseline`, /// `noop`, `auto`, ...) exist only on replies: the daemon decides those. @@ -239,26 +239,6 @@ pub enum Op { #[serde(rename = "fork")] id: String, }, - /// Safe Mode: forks one checkout and mounts it directly at the repo - /// root for the session's duration (see `dry_run` in `.acyclic/config.toml`). - SessionFork { - session_id: String, - }, - /// Safe Mode: unmounts the session's shadow mount, commits its overlay, - /// and returns the resulting diff without touching the real tree yet. - /// The fork is held pending `SessionApply`/`SessionDiscard`. - SessionResolve { - session_id: String, - }, - /// Safe Mode: applies a `SessionResolve`d session's changes to the real - /// tree (the deferred half of promote). - SessionApply { - session_id: String, - }, - /// Safe Mode: discards a `SessionResolve`d session without applying it. - SessionDiscard { - session_id: String, - }, } fn default_fork_count() -> u32 { @@ -313,8 +293,6 @@ pub enum Reply { Summary(SummaryInfo), Forks(Vec), Promote(PromoteInfo), - /// A `SessionResolve`d session awaiting `SessionApply`/`SessionDiscard`. - SessionPending(SessionPendingInfo), } /// One turn's summary, and where it came from. @@ -335,12 +313,6 @@ pub struct SummaryInfo { pub lead_ms: Option, } -#[derive(Debug, Serialize, Deserialize)] -pub struct SessionPendingInfo { - pub session_id: String, - pub diff: Vec, -} - #[derive(Debug, Serialize, Deserialize)] pub struct ForkEntry { pub id: String, @@ -419,7 +391,7 @@ pub struct StatusInfo { pub unpublished: u64, pub store_bytes: u64, pub repo_root: String, - /// Mount provider this daemon would use for forks and Safe Mode. + /// Mount provider this daemon would use for forks. #[serde(default)] pub mount_provider: String, #[serde(default)] diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index efdf1a5..0e21b5b 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -8,9 +8,7 @@ use std::time::{Duration, Instant}; use crate::ipc; use crate::proto; use acyclic::config::Config; -use acyclic::fork::{ - self, ForkMode, MountCapability, PromoteOutcome, SessionResolveOutcome, SharedLocalCheckout, -}; +use acyclic::fork::{self, ForkMode, MountCapability, SharedLocalCheckout}; use acyclic::guard::GuardedMountFilesystem; use acyclic::index::{Attribution, CheckpointKind, CheckpointRow, Index}; use acyclic::merge::{self, Entry}; @@ -21,8 +19,8 @@ use acyclic::store::{Store, StorePaths}; use acyclic::{rewind, EngineError}; use acyclic_fs::model::VolumeConfig; use acyclic_fs::{ - mount_native, mount_native_over_existing, CheckoutMountSource, MountFilesystem, - NativeMountRequest, NativeMountSession, RoutedMountSource, + mount_native, CheckoutMountSource, MountFilesystem, NativeMountRequest, NativeMountSession, + RoutedMountSource, }; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::sync::{Mutex, Notify}; @@ -52,27 +50,6 @@ struct OpenConflict { paths: Vec, } -/// One Safe Mode session: its fork and the shadow mount that projects it -/// directly at the real repo root for the session's duration. Only one can -/// be active at a time -- shadowing is a whole-path substitution, so two -/// sessions can't both shadow the same repo root concurrently. -struct DrySession { - fork_id: String, - session_id: String, - shared: Arc, - base: acyclic::GenerationId, - mount: NativeMountSession, -} - -/// A `SessionResolve`d session awaiting `SessionApply`/`SessionDiscard`. Its -/// overlay is already committed to the store under `generation`; nothing -/// has touched the real tree yet. -struct PendingSession { - generation: acyclic::GenerationId, - base: acyclic::GenerationId, - label: String, -} - /// The one native session projecting every fork through the router. /// Mounted lazily on the first fork, unmounted when the last route goes. struct ForkMount { @@ -169,11 +146,6 @@ fn spawn_speculation( } pub fn run(repo_root: &Path) -> Result<(), String> { - // FIRST, before anything reads through `repo_root`: a Safe Mode shadow - // mount from a crashed daemon leaves the repo root a dead NFS mountpoint - // that wedges every stat/open under it (Config::load, canonicalize, ...). - // The force-unmount acts on the mountpoint path itself without touching - // the dead server, so the real tree reappears before we read the config. let startup = std::time::Instant::now(); let mut phase = std::time::Instant::now(); let mut lap = |name: &str| { @@ -187,9 +159,6 @@ pub fn run(repo_root: &Path) -> Result<(), String> { }; rewind::recover_before_repo_open(repo_root).map_err(|error| error.to_string())?; lap("rewind recovery before repo open"); - fork::sweep_stale_dry_session(repo_root); - lap("sweep stale dry-run session"); - let config = Config::load(repo_root).map_err(|error| error.to_string())?; lap("config load"); let stores_root = config.store_dir.as_ref().map(PathBuf::from); @@ -241,7 +210,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { } if !mounts.available { eprintln!( - "{NAME} daemon: mounts unavailable ({}): forks fall back to copies, Safe Mode is off", + "{NAME} daemon: mounts unavailable ({}): forks fall back to copies", mounts.reason.as_deref().unwrap_or("unknown reason") ); } @@ -263,8 +232,6 @@ pub fn run(repo_root: &Path) -> Result<(), String> { router: Arc::new(RoutedMountSource::new()), session: None, })), - dry_session: Arc::new(Mutex::new(None)), - pending: Arc::new(Mutex::new(HashMap::new())), spec, live_sessions: Arc::new(Mutex::new(HashSet::new())), last_activity: Arc::new(Mutex::new(Instant::now())), @@ -284,7 +251,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { #[derive(Clone)] struct Server { - /// Probed once at start: decides fork mode and gates Safe Mode. + /// Probed once at start to decide whether forks mount or materialize. mounts: MountCapability, handle: PipelineHandle, index_db: PathBuf, @@ -294,10 +261,6 @@ struct Server { shutdown: Arc, forks: Arc>>, fork_mount: Arc>, - /// The one active Safe Mode session shadow-mounted at `repo_root`, if any. - dry_session: Arc>>, - /// Sessions that resolved (committed) but haven't been applied/discarded. - pending: Arc>>, /// The speculation scheduler, when it is enabled. `None` makes every /// call site a no-op, so the default path costs nothing. spec: Option>, @@ -368,13 +331,11 @@ impl Server { } /// True when nothing has needed this daemon for `idle`: no request, no - /// open session, no live fork, no Safe Mode session, nothing pending. + /// open session, and no live fork. async fn idle_for(&self, idle: Duration) -> bool { self.last_activity.lock().await.elapsed() >= idle && self.live_sessions.lock().await.is_empty() && self.forks.lock().await.is_empty() - && self.dry_session.lock().await.is_none() - && self.pending.lock().await.is_empty() } /// The store's size on disk, from a cache that a background walk @@ -717,9 +678,6 @@ impl Server { .session_started(session_id.clone(), host) .await .map_err(stringify)?; - if self.config.dry_run { - self.session_fork(session_id).await?; - } Ok(proto::Reply::Unit) } proto::Op::SessionEnd { session_id } => { @@ -772,13 +730,6 @@ impl Server { if let Some(spec) = self.spec.as_ref() { spec.shutdown().await; } - // Unmount an active Safe Mode shadow first: it sits directly - // on the real repo root, so this must never be left mounted - // once the daemon that owns it is gone. - if let Some(mut session) = self.dry_session.lock().await.take() { - let _ = tokio::task::block_in_place(|| session.mount.stop()); - } - self.pending.lock().await.clear(); // Detach the fork session before the pipeline goes away: its // callback runtimes reach into the shared checkouts. self.forks.lock().await.clear(); @@ -989,159 +940,9 @@ impl Server { ), )) } - proto::Op::SessionFork { session_id } => { - self.session_fork(session_id).await?; - Ok(proto::Reply::Unit) - } - proto::Op::SessionResolve { session_id } => { - self.invalidate(&crate::speculate::Cause::Session(session_id.clone())); - let mut slot = self.dry_session.lock().await; - let session = slot - .take() - .filter(|session| session.session_id == session_id) - .ok_or_else(|| format!("no active Safe Mode session {session_id}"))?; - drop(slot); - // Unmount first: the real tree must reappear before we ask - // the engine to touch it, and no new writes can race the - // commit below. - let DrySession { - fork_id, - session_id, - shared, - base, - mut mount, - } = session; - tokio::task::block_in_place(|| mount.stop()) - .map_err(|error| format!("unmount: {error:?}"))?; - let label = format!("safe mode session {fork_id}"); - let outcome = self - .handle - .resolve_session(Arc::clone(&shared), base, label.clone()) - .await - .map_err(stringify)?; - match outcome { - SessionResolveOutcome::NoChanges => { - Ok(proto::Reply::SessionPending(proto::SessionPendingInfo { - session_id, - diff: Vec::new(), - })) - } - SessionResolveOutcome::Resolved { generation } => { - let changes = self - .handle - .diff(base, generation) - .await - .map_err(stringify)?; - self.pending.lock().await.insert( - session_id.clone(), - PendingSession { - generation, - base, - label, - }, - ); - Ok(proto::Reply::SessionPending(proto::SessionPendingInfo { - session_id, - diff: changes.into_iter().map(diff_entry).collect(), - })) - } - SessionResolveOutcome::Conflict { message } => Err(message), - } - } - proto::Op::SessionApply { session_id } => { - let mut pending = self.pending.lock().await; - let session = pending - .remove(&session_id) - .ok_or_else(|| format!("no resolved Safe Mode session {session_id}"))?; - drop(pending); - let outcome = self - .handle - .apply_session(session.generation, session.base, session.label) - .await - .map_err(stringify)?; - match outcome { - PromoteOutcome::Promoted { - generation, - old_tree, - } => Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic::generation_hex(generation), - old_tree: old_tree.map(|path| path.display().to_string()), - warning: "reload your editor: open files still point at the replaced tree" - .into(), - replayed_paths: 0, - merged_files: 0, - conflicts: Vec::new(), - fork_path: None, - kept_mainline: Vec::new(), - mainline_moved: false, - })), - PromoteOutcome::Conflict { message } => Err(message), - } - } - proto::Op::SessionDiscard { session_id } => { - self.pending.lock().await.remove(&session_id); - Ok(proto::Reply::Unit) - } } } - /// Forks one checkout and shadow-mounts it directly at `repo_root` for - /// `session_id`'s duration (Safe Mode's session redirection). Only one - /// Safe Mode session can be active per repo at a time. - async fn session_fork(&self, session_id: String) -> Result<(), String> { - if !self.mounts.available { - return Err(format!( - "Safe Mode needs a mount provider and this host has none ({}).\n{}", - self.mounts.reason.as_deref().unwrap_or("unknown reason"), - fork::mount_setup_hint() - )); - } - if self.dry_session.lock().await.is_some() { - return Err("a Safe Mode session is already active for this repo".to_owned()); - } - let seed = self.handle.fork().await.map_err(stringify)?; - let shared = Arc::clone(&seed.shared); - let config = seed.config; - let guarded_paths = self.config.guarded_paths.clone(); - let volume_id = seed.volume_id; - let destination = self.repo_root.clone(); - let mount = tokio::task::block_in_place(move || { - let source = CheckoutMountSource::new(shared, config) - .map_err(|error| format!("mount source: {error:?}"))?; - let source: Arc = - if GuardedMountFilesystem::is_active(&guarded_paths) { - Arc::new(GuardedMountFilesystem::new( - Arc::new(source), - &guarded_paths, - )) - } else { - Arc::new(source) - }; - mount_native_over_existing( - NativeMountRequest { - mount_id: acyclic::MountId::new(), - volume_id, - destination, - writable: true, - }, - source, - ) - .map_err(|error| format!("shadow mount: {error:?}")) - })?; - *self.dry_session.lock().await = Some(DrySession { - fork_id: short_id(), - session_id, - shared: seed.shared, - base: seed.base, - mount, - }); - // The fork now shadows the real repo root: suspend mainline capture - // until resolve/apply, or the pipeline watcher captures the shadow's - // content and the mount lifecycle instead of real-tree mutations. - self.handle.set_shadowed(true).await.map_err(stringify)?; - Ok(()) - } - /// Copy-mode forks: materialize the base generation into a real /// directory per fork. Same ids, lifecycle, and promote semantics as /// mounted forks; creation is O(tree) instead of O(1). diff --git a/crates/acyclic/tests/fork.rs b/crates/acyclic/tests/fork.rs index ab0e815..8c4e3cf 100644 --- a/crates/acyclic/tests/fork.rs +++ b/crates/acyclic/tests/fork.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use std::time::Duration; use acyclic::config::Config; -use acyclic::fork::{PromoteOutcome, SessionResolveOutcome}; +use acyclic::fork::PromoteOutcome; use acyclic::index::{Attribution, CheckpointKind, Index}; use acyclic::pipeline::{self, PipelineHandle, State}; use acyclic::store::{Store, StorePaths}; @@ -183,75 +183,6 @@ fn promote_lands_fork_changes() { rig.finish(); } -/// Safe Mode: `resolve_session` + `apply_session` together must land a fork's -/// changes identically to promote's single-shot version. -#[test] -fn resolve_then_apply_session_lands_fork_changes() { - let rig = Rig::start(); - let (diffable_base, outcome) = rig.runtime.block_on(async { - let seed = rig.handle.fork().await.expect("fork"); - write_in_fork(&seed, "/fork-note.txt", b"written in fork\n").await; - let resolved = rig - .handle - .resolve_session( - Arc::clone(&seed.shared), - seed.base, - "safe-mode session".into(), - ) - .await - .expect("resolve_session"); - let SessionResolveOutcome::Resolved { generation } = resolved else { - panic!("expected Resolved, got {resolved:?}"); - }; - let diff = rig.handle.diff(seed.base, generation).await.expect("diff"); - let outcome = rig - .handle - .apply_session(generation, seed.base, "safe-mode session".into()) - .await - .expect("apply_session"); - (diff, outcome) - }); - - // The pre-apply diff already shows the new file, before anything landed. - assert!(diffable_base - .iter() - .any(|change| change.path == Path::new("fork-note.txt"))); - - let PromoteOutcome::Promoted { old_tree, .. } = outcome else { - panic!("expected Promoted, got {outcome:?}"); - }; - assert!(old_tree.is_some(), "a real change must swap the tree"); - assert_eq!( - std::fs::read(rig.repo_path().join("fork-note.txt")).expect("landed file"), - b"written in fork\n" - ); - assert_watcher_recovers_after_swap(&rig); - rig.finish(); -} - -/// Safe Mode: discarding a resolved session (never calling `apply_session`) -/// must leave the real tree completely untouched. -#[test] -fn resolved_session_left_unapplied_leaves_zero_trace() { - let rig = Rig::start(); - rig.runtime.block_on(async { - let seed = rig.handle.fork().await.expect("fork"); - write_in_fork(&seed, "/fork-note.txt", b"written in fork\n").await; - let resolved = rig - .handle - .resolve_session( - Arc::clone(&seed.shared), - seed.base, - "safe-mode session".into(), - ) - .await - .expect("resolve_session"); - assert!(matches!(resolved, SessionResolveOutcome::Resolved { .. })); - }); - assert!(!rig.repo_path().join("fork-note.txt").exists()); - rig.finish(); -} - /// P2: promoting an untouched fork lands nothing and touches nothing. #[test] fn promote_of_untouched_fork_is_noop() { diff --git a/docs/design/00-overview.md b/docs/design/00-overview.md index 76d9977..585ff47 100644 --- a/docs/design/00-overview.md +++ b/docs/design/00-overview.md @@ -25,7 +25,6 @@ Each launch is one engine increment plus one coherent story, ordered by dependen | 1 | Rewind | Merkle snapshot store + host hooks | Never fear letting the agent loose | [01-rewind.md](01-rewind.md) | | 2 | Timeline | Turn-linked metadata index | The repo at any point in the conversation | [02-timeline.md](02-timeline.md) | | 3 | Forks | Copy-on-write materialization | N parallel attempts, pick the winner | [03-forks.md](03-forks.md) | -| 4 | Safe Mode | Session redirection + write interposition | Agents on the codebase, not agents' mistakes in it | [04-safe-mode.md](04-safe-mode.md) | | 5 | Monorepo | Merkle-aware content + symbol index | The repo that finally works with agents | [05-monorepo.md](05-monorepo.md) | Cross-cutting, not a launch: **[Speculation](08-speculation.md)** — the daemon computes what the agent is about to ask for (the session brief, a turn summary) in the time when nobody is waiting. Off by default. @@ -33,7 +32,6 @@ Cross-cutting, not a launch: **[Speculation](08-speculation.md)** — the daemon Sequencing rationale: - Launches 1–2 ship on the cheap engine (hooks + snapshot store) without committing to how forks work. The hard CoW decision only becomes due at Launch 3. -- Launch 4 depends on Launch 3's fork engine (dry-run and scratch trees are fork features). - Launch 5 is dependency-independent (separate index engine) — pull it forward if monorepo teams become the target buyer. ## The load-bearing decision @@ -45,14 +43,13 @@ Second load-bearing constraint, from compliance: **purge-through-history and sna ## Strategic risks (from the design review) 1. **Launch 1 collides with native host features.** Claude Code ships its own checkpoint/rewind. Differentiation must be the headline, not fine print: we capture what Bash did (installs, migrations, generated files), untracked/gitignored state, cross-session persistence, cross-host consistency — and checkpoints become forks. Expect hosts to keep commoditizing the basic rewind; the moat is the Merkle/CoW engine and Launches 3/5. -2. **Unverified host-API assumptions** — validate before public promises: (a) transparent tool interception for indexed search (hook rewrite limits differ per host); (b) session redirection for dry-run (path display, git status confusion); (c) checkpoint alignment in hosts without lifecycle hooks. +2. **Unverified host-API assumptions** — validate before public promises: (a) transparent tool interception for indexed search (hook rewrite limits differ per host); (b) checkpoint alignment in hosts without lifecycle hooks. 3. **Naming**: repo is `graphcoder-plugin`, CLI is `acyclic`, and Graphcoder is a different product on the roadmap. Resolve before launch. ## Settled since this was written 1. **Fork engine mechanism** — decided the opposite way to the lean recorded here. Mounts shipped; there is no reflink or `clonefile` path in the codebase, and the fallback when no mount provider is available is a **full copy**, not a reflink. Forks are routes inside one kernel mount rather than N mounts. See `implementation-forks.md`. 2. **Checkpoint alignment in hosts without lifecycle hooks** — resolved by the MCP adapter plus the `auto_checkpoint_idle_ms` timer. MCP is a second adapter shape this doc's thesis line does not yet mention. -3. **Session redirection for dry-run** — built and tested as Safe Mode, mount-only. ## Open questions (not yet settled) @@ -61,5 +58,4 @@ Second load-bearing constraint, from compliance: **purge-through-history and sna 3. **Naming**: repo is `graphcoder-plugin`, CLI is `acyclic`, and Graphcoder is a different product on the roadmap. Still unresolved. 4. **Repo visibility** — this repo is private while its `acyclic-fs` dependency is public, which blocks the curl installer, the attestations, and the open-source claim. Carried in `06-installation.md` and `07-compliance.md`; it belongs at overview level because it gates positioning, not just packaging. 5. **The upstream retention dependency.** GC, purge and enforced retention all wait on an `acyclic-fs` retention-release fact that does not exist. This is the single largest gap between the compliance story and the code, and it is not ours to close. -6. **Degrade or refuse without a mount provider.** Forks silently fall back to full copies; Safe Mode refuses to start. A repo with `dry_run = true` checked in therefore runs against the real tree on a host without mounts. Which of those two behaviours is right has never been decided as a policy. 7. **What remains unvalidated at scale.** Store performance on a real 10GB tree is still the first thing to validate, as it was when this doc was written. The latency gate runs 20k files / 256 MB, and `init` baseline capture runs ~230 s/GiB. diff --git a/docs/design/01-rewind.md b/docs/design/01-rewind.md index 6687e99..dcf23b8 100644 --- a/docs/design/01-rewind.md +++ b/docs/design/01-rewind.md @@ -116,7 +116,7 @@ leads with what it can't do: 4. **Hardcoded constants that are really policy** — mutation batch size, maximum capture paths, extent spans, watch queue depth, prompt excerpt bytes. None are configurable and none are documented. -5. **Excluded paths are invisible to forks and Safe Mode sessions.** A secret +5. **Excluded paths are invisible to forks.** A secret kept out of the store is also absent from every fork, which is either the correct safety property or a broken build, depending on the secret. *No lean.* diff --git a/docs/design/03-forks.md b/docs/design/03-forks.md index 7457dd3..5ccd9f6 100644 --- a/docs/design/03-forks.md +++ b/docs/design/03-forks.md @@ -73,7 +73,7 @@ is what exists, including where it diverged from the plan. ## Notes - Fork orchestration of subagents is uniquely plugin-shaped — it must live inside the host. -- Filesystem-layer enforcement for Launch 4's guarded paths arrives with the mount option. This turned out to be the decisive argument: the mount shipped and reflinks never did, and Safe Mode refuses to start without a mount provider. +- Filesystem-layer enforcement for guarded paths arrives with the mount option. Copy-mode forks cannot enforce this policy at the filesystem boundary. ## Open questions (not yet settled) diff --git a/docs/design/04-safe-mode.md b/docs/design/04-safe-mode.md deleted file mode 100644 index 98f361d..0000000 --- a/docs/design/04-safe-mode.md +++ /dev/null @@ -1,122 +0,0 @@ -# Launch 4 — Safe Mode - -Fork-engine features for trust-sensitive teams. Depends on Launch 3. - -Status: **built, mount-dependent.** Claims below are tagged *shipped*, -*not built*, or *diverged* where what exists differs from what was planned. - -## Features - -11. **Dry-run mode** — the whole session runs against a fork; nothing hits the - real tree until the dev approves the final diff. *Shipped, off by default.* -12. **Ephemeral scratch trees** — disposable full-repo copies that vanish on - session end. *Diverged: the auto-drop machinery exists and runs at session - end, but nothing creates a tagged scratch tree. There is no user-facing - scratch verb and no caller sets the session id, so this feature has a - working destructor and no constructor.* -13. **Guarded paths** — writes the agent can't make, enforced at the filesystem - layer rather than by prompt hope. *Shipped for mounted forks and Safe Mode - sessions only. See the first open question — outside those, it is a no-op.* - -## User journey - -The journey as originally written is still the target. Two steps do not work -as described today: - -1. Maya's team checks in `.acyclic/config.toml`: dry-run on, `.env` and - `migrations/` guarded. Everyone who clones inherits the policy. *Shipped.* -2. A teammate starts a session rooted in a fork; paths look normal, the real - tree is untouched. *Shipped, if the host has a mount provider.* -3. The agent grabs a scratch tree for a destructive codemod and lets it vanish. - *Not built — see feature 12.* -4. The agent tries to edit `.env`; the write is refused at the interposition - layer. *Shipped inside the session. Note the macOS caveat below: the refused - write can still report success to the shell.* -5. The session ends with a final diff the teammate reviews and approves. - *Diverged: `session-resolve`, `session-apply` and `session-discard` are - hidden CLI-only verbs. No host adapter wires them, they are not MCP tools, - and session end does not resolve the session — the shadow mount stays up.* - -## What was built - -- **Session redirection** — *shipped.* The session is rooted in a shadow mount. -- **Approval-gated apply** — *shipped as CLI verbs, unwired.* See journey 5. -- **Write interposition for guarded paths** — *diverged, and in the opposite - order to the plan.* The plan was to ship hook-level blocking first and - upgrade with the mount. Hook-level was never built: the hook contract is to - never block and always exit 0, and it never reads `guarded_paths`. Only the - mount-level guard exists, so enforcement and Safe Mode arrived together. -- **Policy configuration** — *shipped, minus scratch-tree limits*, which have - no config key because there are no scratch trees. -- **Scratch-tree lifecycle** — *diverged.* There is no janitor process. - Session-owned forks are dropped best-effort in the session-end handler, and a - crashed daemon's shadow mount is reaped opportunistically on the next CLI - invocation — an on-demand sweep, not a cycle. - -### Constraints that were never written down - -- **One Safe Mode session per repo.** A second is refused outright. -- **Apply requires an unmoved mainline.** If the real tree moved during the - session, the choice is rewind to base or start again — the doc promised an - atomic apply with no such condition. -- **Mainline capture is suspended for the whole session.** A dry-run session - records no mainline checkpoints. -- **Guarded paths are excluded from the session diff entirely**, so the - approval view never shows them. -- **The guard surface is wider than "writes"**: it covers rename and hard-link - on both source and destination, and unconditionally refuses AppleDouble - sidecars. -- **Reflink acceleration is disabled whenever any guard is configured**, so - guarded repos silently fall back to read-and-write copies. - -## Acceptance criteria - -- A dry-run session is indistinguishable from a normal one to the agent. - **Unverified** — there is no live-host Safe Mode test; the suite drives the - CLI with a synthetic host name. -- Guarded-path writes are refused with a legible error the agent can act on. - **Partially met, with a known correctness caveat**: on macOS a refused write - can still return exit 0 to the shell because of NFS write-back caching, so - the agent may see success where the filesystem saw refusal. -- Rejected sessions leave zero trace on the real tree. **Met** — and the - converse is now explicit: a crash discards all session work. Zero trace and - zero recovery are the same property. -- Orphaned scratch trees collected within one janitor cycle. **Not met** — no - janitor exists. - -## Open questions (not yet settled) - -1. **Guarded paths are a no-op in the default configuration.** With - `dry_run = false` (the default) and no fork, `guarded_paths` is never - consulted and the agent can write `.env` freely. A team that checks in - guarded paths and reads the README table has every reason to believe - otherwise. This is the highest-severity gap in this launch. *No lean — it - is either a documentation fix or a feature.* -2. **Silent degradation on a host without mounts.** Safe Mode refuses to start, - the session-start op fails, and the hook prints to stderr and exits 0. A - repo with `dry_run = true` checked in therefore runs against the real tree - with no visible failure. *Current lean: this should be loud — a refusal to - proceed rather than a warning nobody reads.* -3. **Copy-mode forks are unguarded**, because the guard wraps mount routes - only. On a mount-less host guarded paths are unenforced even for explicit - forks. *No lean.* -4. **Do the approval verbs get wired into hosts?** They are the visible half of - the feature and are currently reachable only by someone who reads the hidden - CLI surface. *Current lean: yes, and Safe Mode should not be promoted until - they are.* -5. **Is the NFS write-back caveat acceptable?** It weakens the acceptance - criterion it contradicts, and the failure direction is the dangerous one — - the agent believes a guarded write succeeded. *No lean.* -6. **Should scratch trees be built or dropped from the feature list?** The - destructor exists; the constructor does not. *Current lean: drop the claim - until there is a verb.* -7. **Hardcoded constants that are really policy**: the 16-fork cap, the 5s reap - probe, and the 2s pre-tool wait after which the tool proceeds uncheckpointed. - -## Open design risks - -- Session redirection fights host assumptions (path display, git status - confusion). Still unvalidated against a real host. -- Hook-level guarded paths may not be worth shipping alone. **This was - settled by not doing it** — the mount-level guard shipped instead, which - means guarded paths inherit every one of Safe Mode's platform constraints. diff --git a/docs/design/05-monorepo.md b/docs/design/05-monorepo.md index cc753a6..1410f52 100644 --- a/docs/design/05-monorepo.md +++ b/docs/design/05-monorepo.md @@ -73,7 +73,7 @@ which reads as though Launch 5 exists. 3. **`exclude` blinds any future index.** Excluded paths never enter a checkpoint, so a Merkle-aware index can never answer for them, and the answer it gives will be silently incomplete rather than refused. *No lean.* -4. **Forks and Safe Mode sessions do not see excluded paths either**, which +4. **Forks do not see excluded paths either**, which bears directly on "searches against a fork must answer from that tree's state". *No lean.* 5. **Does the name change?** Calling this "the index engine" collides with the diff --git a/docs/design/06-installation.md b/docs/design/06-installation.md index 1d7a202..59accf6 100644 --- a/docs/design/06-installation.md +++ b/docs/design/06-installation.md @@ -206,7 +206,7 @@ The two adapter shapes above — lifecycle hooks, and JSON-based MCP registratio ## Configuration -- Per-repo: `.acyclic/config.toml` — checked in, so teams share policy: checkpoint granularity, guarded paths, dry-run default, store size caps, retention TTLs. +- Per-repo: `.acyclic/config.toml` — checked in, so teams share policy: checkpoint granularity, guarded paths, store size caps, retention TTLs. - Per-machine: `~/.config/acyclic/` — defaults. - Zero config is a supported state — defaults are safe everywhere. diff --git a/docs/design/07-compliance.md b/docs/design/07-compliance.md index e9a5432..1d277df 100644 --- a/docs/design/07-compliance.md +++ b/docs/design/07-compliance.md @@ -113,9 +113,8 @@ Two honest limits on that pitch: - Hosts without a prompt hook — Claude Desktop, VS Code, OpenCode — get checkpoints without turn linkage, so the headline evidence is thinner exactly where the adapter is thinnest. -- The "a human approved it before it reached the real tree" half rests on Safe - Mode, which is off by default, refuses to start without a mount provider, and - whose approval verbs are hidden CLI-only and wired into no host. +- V1 does not interpose a human approval gate on ordinary agent writes. Explicit + forks isolate work only when the user or host chooses that workflow. ## When the cloud arrives (v2) @@ -149,4 +148,4 @@ as a silent default — and the engine remains fully usable with the cloud off. claiming it.* 6. **Is `exclude` enough as the only shipped control?** It is prefix-based, start-time-bound, and non-retroactive, and excluded paths are invisible to - forks and Safe Mode sessions. *No lean.* + forks. *No lean.* diff --git a/docs/design/implementation-forks.md b/docs/design/implementation-forks.md index 8c76d4c..8c9b7b8 100644 --- a/docs/design/implementation-forks.md +++ b/docs/design/implementation-forks.md @@ -3,7 +3,7 @@ Ships `03-forks.md`: N-way local forks, pick the winner. Built on fs's native mounts — the resolution of the plan's open CoW question is **mounts, not reflinks**: fs now ships qualified FUSE-T/FUSE/ProjFS drivers and the mount -path is the strategic asset (lazy hydration, Launch 4's filesystem-level +path is the strategic asset (lazy hydration and filesystem-level guarded-path enforcement). Reflinks remain a fallback if mount UX disappoints. ## How a fork works (the fs wiring, verified against fsd's usage) diff --git a/docs/design/implementation-merge.md b/docs/design/implementation-merge.md index 2f39f48..c8c1630 100644 --- a/docs/design/implementation-merge.md +++ b/docs/design/implementation-merge.md @@ -32,8 +32,7 @@ plan. **Out (later launches, listed so nobody re-litigates them mid-build):** rename detection; semantic or AST merges; markers written into the -*mainline*; Safe Mode `apply_session` onto a moved mainline (keeps the -unmoved-mainline rule); merging directory-ancestry overlaps; live rebase +*mainline*; merging directory-ancestry overlaps; live rebase of a fork while the mainline moves. ## Definitions @@ -206,7 +205,7 @@ before anything is written. One pipeline request `BuildGeneration { from: GenerationId, entries }`: scratch checkout at `from` (`scratch_checkout` exists), write each entry through the SDK checkout API the fork tests already use, `checkpoint()` -it unpublished (as `resolve_session` does), `record_generation` it. +it unpublished, then `record_generation` it. M = `BuildGeneration(H, merged ∪ taken)`; R = `BuildGeneration(M, conflicted)`. No head movement, no tree writes. @@ -263,7 +262,7 @@ conflict state, proto fields, CLI lines, marker-scan on re-promote. Skill text update. **C3 — acceptance.** `merge.sh` changes below, run in mount mode on macOS -and copy mode on Linux, plus `forks.sh`, `safe-mode.sh`, and the journey +and copy mode on Linux, plus `forks.sh` and the journey suites unchanged. Update `spec-forks.md`'s out-of-scope line, its stale M4/M5 rows, and the c905be4 "discard" wording in the same commit. @@ -380,10 +379,10 @@ Deliberate divergences: fork: through the shared checkout for a mount fork (route detached during promote, re-attached after), via `restore_path_into` for a copy fork. -- **Rebased forks land by checkpoint-and-swap.** A rebased mount fork's - checkout still sits on the head it was cut from, so an optimistic +- **Rebased forks land through the copy-fork snapshot/apply path.** A rebased + mount fork's checkout still sits on the head it was cut from, so an optimistic commit against the new head would conflict. `ForkState.rebased` routes - such forks through the same resolve/apply pair copy forks use. + such forks through the same snapshot/apply path copy forks use. - **Subtree copy and removal are iterative.** The fs facade's futures are large enough that three nested levels overflowed the pipeline thread's 2 MiB stack in a debug build. Both walks use an explicit work @@ -419,9 +418,8 @@ Deliberate divergences: unmoved mainline used to land by whole-tree swap, which replaced the repo directory and needed the skill's `cd "$PWD"` step. Promote now always goes through the merge path: with an unmoved head the plan is - "take every fork path" and they are written in place. Safe Mode's - `session-apply` is the only remaining swap. `merge.sh` G11 asserts - the repo inode survives a promote. + "take every fork path" and they are written in place. `merge.sh` G11 + asserts the repo inode survives a promote. - **Diff output marks gitignored paths** with a `(gitignored)` suffix and a `(K gitignored)` count so the skill's smallest-diff rule can ignore cache and build noise; the paths stay listed because rewind @@ -451,14 +449,11 @@ Deliberate divergences: prints the wait-path p95 as information. - **Path tracing.** `ACYCLIC_TRACE=1` makes the CLI, hook, daemon, and pipeline log every branch taken and its cost (client connect/spawn, - op dispatch and reply, WAIT vs ENQUEUE checkpoint, shadowed noop, + op dispatch and reply, WAIT vs ENQUEUE checkpoint, recovery baseline, watcher drain polls/batches/stop reason, snapshot, index, publish, promote mode and outcome, merge plan counts, root hints). Daemon lines land in the store's `daemon.log`. -- **Safe Mode S9 race.** After an empty session resolve unmounted the - shadow and installed a fresh watcher, a late mount-teardown event for - the repo root itself reached the next drain, and the engine refused a - mutation targeting the volume root. Root-targeted hints are now +- **Root-targeted watcher hints.** Root-targeted hints are now stripped before capture: metadata-only ones dropped, structural ones (root created/removed/renamed, i.e. a mount came or went) trigger a fresh watcher and a recovery baseline instead of a failed request. diff --git a/docs/design/implementation-rewind.md b/docs/design/implementation-rewind.md index 3cdef18..c41a534 100644 --- a/docs/design/implementation-rewind.md +++ b/docs/design/implementation-rewind.md @@ -123,12 +123,12 @@ Built as `acyclic hook ` + `acyclic install ` (the installer embeds Shipped: - Acceptance: `exclusions.sh` (declared paths never captured, noop on excluded-only edits, restore refused, rename into an excluded prefix scrubbed, rewind carries the live copies, pre-rule history untouched) and `growth.sh` (60 distinct checkpoints cost ~52 KB each, identical on a 4x larger tree: per-checkpoint overhead is tree pages, never a tree copy). Journey/soak/crash/latency were already green; the migration-script scenario is journey.sh's Bash side-effect step. -- Snapshot exclusions (`exclude` in config): `acyclic-engine/src/exclude.rs`. Watcher hints at or under a rule are dropped before capture; renames across the boundary re-examine the uncovered side; a capture hinted at an ancestor (or the baseline) is followed by a checkout scrub before the generation is checkpointed; a full rewind (and promote / Safe Mode apply, which share `rewind::execute`) moves the live excluded paths into the new tree before the swap, journaled as a `Carrying` phase so kill -9 at any point returns them. +- Snapshot exclusions (`exclude` in config): `acyclic-engine/src/exclude.rs`. Watcher hints at or under a rule are dropped before capture; renames across the boundary re-examine the uncovered side; a capture hinted at an ancestor (or the baseline) is followed by a checkout scrub before the generation is checkpointed; a full rewind moves the live excluded paths into the new tree before the swap, journaled as a `Carrying` phase so kill -9 at any point returns them. - Retention: `status` reports store size, trash is TTL-pruned. **No fs GC, no purge in v1**, and this is a finding, not an omission: at sdk `8eced48`, `collect_local_garbage` keeps only authority heads plus retention facts (`RetentionKind::{Checkpoint,Pin,ForkBase}`), `retain_workspace_generation` is create-only (no release fact exists), and `prove_generation_closure` does not follow `GenerationRoot::parents`. Running GC would delete every checkpoint but the head; pinning every checkpoint first would make the store append-only forever. Purge cannot remove bytes from a retained generation for the same reason. **Upstream ask:** a retention-release fact (mirror of `encode_workspace_deleted` for retention authorities) honoured by the collector, plus a bulk "retain these N generations" call. With that, the plugin's policy is straightforward: pin what the TTL keeps, release the rest, collect. - Release engineering: `deny.toml` + a `deny` CI job (licenses inside a permissive allowlist, advisories, sources); an SPDX SBOM generated from `Cargo.lock` per target in `release.yml`, attested against each binary with `actions/attest-sbom`, one copy attached to the GitHub release and listed in `SHA256SUMS`; `scripts/install.sh` (verified download into `~/.local/bin`, `file://` base URL for offline tests); `scripts/install-smoke.sh` runs it on a bare Debian container and drives init → checkpoint → rewind, exclusions included. npm `@acyclic-labs/plugin` 0.0.1 is live. - Docs: README leads with the differentiators, documents `exclude`, and states the retention stance and caveats. -Still open, by decision: the public name (`acyclic` vs graphcoder), and cutting the first `v*` tag through `release.yml`; both are the owner's call. Exclusions do not reach forks or Safe Mode sessions (they are served from generations), documented as a caveat. +Still open, by decision: the public name (`acyclic` vs graphcoder), and cutting the first `v*` tag through `release.yml`; both are the owner's call. Exclusions do not reach forks (they are served from generations), documented as a caveat. --- diff --git a/docs/design/spec-forks.md b/docs/design/spec-forks.md index 24920a8..5226b7e 100644 --- a/docs/design/spec-forks.md +++ b/docs/design/spec-forks.md @@ -77,8 +77,7 @@ Unmounts, discards the overlay, removes the workspace dir. Unknown id → error. onto the real tree one at a time with the same atomic single-path restore, checkpointed and published; a `manual` row labeled `promote (N path(s) written in place)` records it. The repo - directory is never replaced by a promote (Safe Mode `session-apply` - still swaps and says so). Gitignored files included. + directory is never replaced by a promote. Gitignored files included. ### Daemon stop / crash Stop unmounts all forks and removes workspace dirs before the pipeline @@ -107,6 +106,6 @@ remove) at next start, before the store opens. | U2: stale sweep removes dirs | `fork.rs::sweep_removes_stale_directories` | unit | ✅ | Out of scope (documented): rename detection, semantic merges, conflict -markers on the mainline, Safe Mode `apply_session` onto a moved mainline, -fork persistence across daemon restarts, subagent orchestration, per-fork +markers on the mainline, fork persistence across daemon restarts, +subagent orchestration, per-fork port/env provisioning. diff --git a/docs/windows-verification.md b/docs/windows-verification.md index 0755be1..e00b591 100644 --- a/docs/windows-verification.md +++ b/docs/windows-verification.md @@ -2,8 +2,7 @@ **Status: verified on real Windows hardware.** Checks 1–8 below were run on Windows 11 (26200) with the MSVC toolchain, against a release build of this -branch. Two capabilities are deliberately *not* offered on Windows — mounted -forks and Safe Mode — for a reason recorded under [Known +branch. Mounted forks have platform-specific limits recorded under [Known limits](#known-limits); everything else behaves as it does on POSIX. `tests/acceptance/windows-smoke.sh` is the executable form of checks 3–7, @@ -52,8 +51,8 @@ That is one decision, not two, and it reaches further than capture. The `ProjFS` provider decodes every entry name it is handed as UTF-16LE, so a name that arrives in any other encoding is *projected as mojibake rather than rejected* — fork route ids passed as raw ASCII came back as six garbage -characters. The Safe Mode guard has the sharper version of the same problem: -it compares configured prefixes byte-for-byte against mount path components, +characters. Guarded fork paths have the sharper version of the same problem: +they compare configured prefixes byte-for-byte against mount path components, and a guard that never matches **fails open**. `crates/acyclic-engine/src/names.rs` exists so there is exactly one place this can be got wrong. @@ -169,15 +168,14 @@ correct. ## Known limits -- **Forks are always copies, and Safe Mode is off.** `ProjFS` mounts and +- **Forks are always copies.** `ProjFS` mounts and projects a fork correctly — the tree appears and reads back fine — but writes into the projection stop at the `ProjFS` local cache and never reach the overlay checkout. A mounted fork therefore looked like it worked while `fork-diff` reported no changes and `promote` landed nothing: it silently ate the work. `mount_capability()` now reports mounts unavailable on Windows, so forks take the copy path, which is verified end to end (write, - `fork-diff`, `promote` all behave). Safe Mode needs a real mount and is - unavailable for the same reason. Revisit if the sdk's `ProjFS` provider + `fork-diff`, `promote` all behave). Revisit if the sdk's `ProjFS` provider gains write-back. - **The tree exchange is not atomic.** `RENAME_EXCHANGE` has no Windows diff --git a/tests/acceptance/run-all.sh b/tests/acceptance/run-all.sh index f6be81e..b1b0834 100755 --- a/tests/acceptance/run-all.sh +++ b/tests/acceptance/run-all.sh @@ -6,7 +6,7 @@ set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" FAILED=0 -SCRIPTS=(journey.sh timeline.sh exclusions.sh growth.sh soak.sh crash.sh latency.sh speculation.sh forks.sh forks-scale.sh merge.sh safe-mode.sh mcp-e2e.sh) +SCRIPTS=(journey.sh timeline.sh exclusions.sh growth.sh soak.sh crash.sh latency.sh speculation.sh forks.sh forks-scale.sh merge.sh mcp-e2e.sh) # The live host-session tests need their CLI + credentials and cost a model # session each; opt in with ACYCLIC_E2E=1. (mcp-e2e.sh is not one of them: # it drives `acyclic mcp` with a scripted client, no host app or model.) diff --git a/tests/acceptance/safe-mode.sh b/tests/acceptance/safe-mode.sh deleted file mode 100755 index 22940e5..0000000 --- a/tests/acceptance/safe-mode.sh +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env bash -# Launch 4 Safe Mode acceptance (spec: docs/design/04-safe-mode.md): session -# redirection through a shadow mount at the repo root, filesystem-level -# guarded paths that hold against arbitrary shell, approval-gated -# apply/discard with zero trace, conflict legibility, and crash sweep. -# Skips (exit 0) when native mounts are unavailable unless -# ACYCLIC_FORKS_REQUIRED=1. -source "$(dirname "${BASH_SOURCE[0]}")/common.sh" - -skip() { - if [ "${ACYCLIC_FORKS_REQUIRED:-0}" = "1" ]; then - fail "$*" - fi - echo "SKIP($(basename "$0")): $*" - exit 0 -} - -# Every mount is torn down on exit even on failure: the shadow mount sits on -# the real repo root, so a leak here poisons every later script. -safe_teardown() { - acy stop >/dev/null 2>&1 || true - sleep 0.3 - if mount | grep -q " $R "; then - umount -f "$R" 2>/dev/null || diskutil unmount force "$R" >/dev/null 2>&1 || true - fi - teardown -} -trap safe_teardown EXIT - -# Environment hygiene (mirrors forks.sh): a kill -9'd daemon anywhere orphans -# its go-nfsv4 helper, and orphaned helpers wedge FUSE-T's tiny shared NFS -# port pool for every later mount — which this script does many of, including -# a crash-recovery remount. Reap acyclic helpers + mounts up front. -if [ "$(uname -s)" = "Darwin" ]; then - pkill -f 'go-nfsv4.*acyclic-fs' 2>/dev/null || true - sleep 0.5 - mount | awk '/fuse-t:\/acyclic-fs|127\.0\.0\.1:\//{print $3}' | while read -r M; do - case "$M" in */acyclic-acceptance.*) umount -f "$M" 2>/dev/null || true ;; esac - done -fi - -shadowed() { mount | grep -q " $R "; } -sha() { shasum -a 256 "$1" | awk '{print $1}'; } -tree_digest() { - (cd "$1" && find . -type f ! -path './.acyclic/*' -print0 | sort -z \ - | xargs -0 shasum -a 256 | shasum -a 256 | awk '{print $1}') -} -# A guarded command must be REFUSED, but on the macOS loopback-NFS transport -# a rejected write can still return exit 0 to the shell (client write-back -# caching acks the write before the server refuses it). So a guard assertion -# must verify the *effect* was prevented, not the command's exit status — -# `must_fail` is only for control-plane commands (session/apply) that reply -# synchronously. Guarded-write refusals are checked by content below. -must_fail() { - local label="$1"; shift - if "$@" 2>"$WORK/err"; then - fail "$label: succeeded but must be refused" - fi -} - -setup_repo -mkdir -p "$R/migrations" "$R/src/nested" -printf 'CREATE TABLE a;\n' > "$R/migrations/001.sql" -printf 'deep\n' > "$R/src/nested/deep.txt" -printf 'ORIGINAL-B\n' > "$R/src/b.rs" -cat >> "$R/.acyclic/config.toml" <<'EOF' -dry_run = true -guarded_paths = [".env", "migrations/"] -EOF -acy init >/dev/null || fail "init" -REAL_BEFORE="$(tree_digest "$R")" -INODE_BEFORE="$(inode "$R")" - -# --- S1: session start shadows the repo root; identical to the agent ------ -if ! OUT="$(acy session-start dry-1 --host acceptance 2>&1)"; then - echo "$OUT" | grep -qi 'mount' && skip "native mounts unavailable: $OUT" - fail "session-start: $OUT" -fi -shadowed || fail "S1: repo root is not shadow-mounted after session-start" -[ "$(cat "$R/src/main.rs")" = "ORIGINAL" ] || fail "S1: shadow does not serve the tree" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S1: gitignored file missing from shadow" -[ "$(cat "$R/migrations/001.sql")" = "CREATE TABLE a;" ] || fail "S1: guarded file unreadable" -[ "$(cat "$R/src/nested/deep.txt")" = "deep" ] || fail "S1: nested path missing" -[ "$(ls "$R" | sort | tr '\n' ' ')" = "$(printf 'migrations src \n' )" ] \ - || fail "S1: listing differs from real tree: $(ls "$R" | tr '\n' ' ')" - -# --- S2: a second Safe Mode session is refused while one is active -------- -must_fail "S2: second session-start" acy session-start dry-2 --host acceptance -grep -qi "already active" "$WORK/err" || fail "S2: refusal not legible: $(cat "$WORK/err")" - -# --- S3: ordinary writes land in the shadow, at every depth --------------- -printf 'MIGRATED\n' > "$R/src/main.rs" || fail "S3: write main.rs" -printf 'deeper\n' > "$R/src/nested/deep.txt" || fail "S3: nested write" -mkdir -p "$R/src/new/dir" || fail "S3: mkdir" -printf 'fresh\n' > "$R/src/new/dir/file.txt" || fail "S3: create in new dir" -rm "$R/src/b.rs" || fail "S3: rm" -head -c 4096 /dev/zero > "$R/generated.bin" || fail "S3: generate artifact" -mv "$R/src/nested/deep.txt" "$R/src/nested/moved.txt" || fail "S3: rename" -[ "$(cat "$R/src/main.rs")" = "MIGRATED" ] || fail "S3: readback main.rs" -[ "$(cat "$R/src/nested/moved.txt")" = "deeper" ] || fail "S3: readback rename" -[ ! -e "$R/src/b.rs" ] || fail "S3: rm not reflected" - -# --- S4: guarded paths refuse EVERY mutation from arbitrary shell --------- -# Each mutation is attempted through plain shell; the exit code is ignored -# (macOS NFS write-back may ack a doomed write), so refusal is proven by the -# guarded content/structure being byte-identical afterward. -try() { "$@" >/dev/null 2>&1 || true; } -try bash -c "printf 'LEAK=1\n' > '$R/.env'" -try bash -c "printf 'LEAK=1\n' >> '$R/.env'" -try bash -c ": > '$R/.env'" -try rm "$R/.env" -try mv "$R/.env" "$R/env.bak" -try bash -c "printf 'x\n' > '$R/tmp.txt' && mv '$R/tmp.txt' '$R/.env'" -try chmod 600 "$R/.env" -try bash -c "printf 'DROP TABLE a;\n' > '$R/migrations/001.sql'" -try bash -c "printf 'x\n' > '$R/migrations/002.sql'" -try mkdir "$R/migrations/sub" -try rm "$R/migrations/001.sql" -try bash -c "rm -rf '$R/migrations'" -try bash -c "mv '$R/generated.bin' '$R/migrations/'" -try mv "$R/migrations" "$R/old-migrations" -rm -f "$R/tmp.txt" -# The refusals were real: guarded content and structure are byte-identical. -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S4: .env changed: $(cat "$R/.env")" -[ "$(cat "$R/migrations/001.sql")" = "CREATE TABLE a;" ] || fail "S4: migration changed" -[ ! -e "$R/migrations/002.sql" ] || fail "S4: file created in guarded dir" -[ ! -e "$R/migrations/sub" ] || fail "S4: dir created in guarded dir" -[ -e "$R/generated.bin" ] || fail "S4: artifact vanished during refused move" -[ -e "$R/migrations" ] || fail "S4: guarded dir vanished" -[ ! -e "$R/env.bak" ] && [ ! -e "$R/old-migrations" ] || fail "S4: guarded rename escaped" -# Reads under guard keep working, and a sibling path (prefix, not guarded) does too. -printf 'ok\n' > "$R/migrations-notes.txt" || fail "S4: prefix-sibling path wrongly guarded" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S4: guarded read broken" - -# --- S5: the real tree is untouched underneath, and hooks keep working ----- -# Checkpoints taken mid-session (hooks fire as normal) must neither wedge the -# daemon nor pollute the mainline timeline with fork content. -acy checkpoint --wait --kind post --session-id dry-1 --tool-name Bash >/dev/null \ - || fail "S5: checkpoint during shadowed session" -STATUS="$(acy status)" -echo "$STATUS" | grep -q "state: ready" || fail "S5: daemon not ready mid-session: $STATUS" - -# --- S6: resolve unmounts, restores the real view, shows the exact diff --- -DIFF="$(acy session-resolve dry-1)" || fail "S6: session-resolve: $DIFF" -shadowed && fail "S6: still shadow-mounted after resolve" -[ "$(inode "$R")" = "$INODE_BEFORE" ] || fail "S6: real root inode changed" -[ "$(tree_digest "$R")" = "$REAL_BEFORE" ] || fail "S6: real tree changed before apply" -echo "$DIFF" | grep -q "^M src/main.rs" || fail "S6: diff missing edit: $DIFF" -echo "$DIFF" | grep -q "^D src/b.rs" || fail "S6: diff missing rm: $DIFF" -echo "$DIFF" | grep -q "^A generated.bin" || fail "S6: diff missing artifact: $DIFF" -echo "$DIFF" | grep -q "^A src/new/dir/file.txt" || fail "S6: diff missing nested create: $DIFF" -echo "$DIFF" | grep -q "^A src/nested/moved.txt" || fail "S6: diff missing rename target: $DIFF" -echo "$DIFF" | grep -q "^D src/nested/deep.txt" || fail "S6: diff missing rename source: $DIFF" -echo "$DIFF" | grep -q "^A migrations-notes.txt" || fail "S6: diff missing sibling: $DIFF" -echo "$DIFF" | grep -q "\.env" && fail "S6: guarded .env in diff: $DIFF" -echo "$DIFF" | grep -q "migrations/" && fail "S6: guarded dir in diff: $DIFF" -echo "$DIFF" | grep -q "session-apply dry-1" || fail "S6: no apply instruction: $DIFF" -must_fail "S6: resolve twice" acy session-resolve dry-1 -must_fail "S6: resolve unknown" acy session-resolve nope - -# --- S7: discard leaves zero trace; a fresh session starts clean ---------- -acy session-discard dry-1 >/dev/null || fail "S7: session-discard" -[ "$(tree_digest "$R")" = "$REAL_BEFORE" ] || fail "S7: discard touched the real tree" -must_fail "S7: apply after discard" acy session-apply dry-1 -acy session-start dry-2 --host acceptance >/dev/null || fail "S7: new session after discard" -shadowed || fail "S7: second session not shadowed" -[ "$(cat "$R/src/main.rs")" = "ORIGINAL" ] || fail "S7: discarded edit leaked into next session" -[ ! -e "$R/generated.bin" ] || fail "S7: discarded artifact leaked into next session" - -# --- S8: a session with no writes resolves as no changes ----------------- -OUT="$(acy session-resolve dry-2)" || fail "S8: resolve empty session" -echo "$OUT" | grep -q "no changes" || fail "S8: expected 'no changes': $OUT" -shadowed && fail "S8: still mounted after empty resolve" -[ "$(tree_digest "$R")" = "$REAL_BEFORE" ] || fail "S8: empty session changed the tree" - -# --- S9: apply lands atomically; guarded content survives ---------------- -acy session-start dry-3 --host acceptance >/dev/null || fail "S9: session-start" -printf 'APPLIED\n' > "$R/src/main.rs" -printf 'brand new\n' > "$R/new.txt" -rm "$R/src/b.rs" -acy session-resolve dry-3 >/dev/null || fail "S9: resolve" -[ "$(cat "$R/src/main.rs")" = "ORIGINAL" ] || fail "S9: real tree changed before apply" -OUT="$(acy session-apply dry-3)" || fail "S9: session-apply: $OUT" -shadowed && fail "S9: mounted after apply" -[ "$(cat "$R/src/main.rs")" = "APPLIED" ] || fail "S9: edit not applied" -[ "$(cat "$R/new.txt")" = "brand new" ] || fail "S9: new file not applied" -[ ! -e "$R/src/b.rs" ] || fail "S9: rm not applied" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S9: guarded .env changed by apply" -[ "$(cat "$R/migrations/001.sql")" = "CREATE TABLE a;" ] || fail "S9: guarded migration changed" -[ -d "$R/.acyclic" ] || fail "S9: repo config lost by apply" -must_fail "S9: apply twice" acy session-apply dry-3 -# The applied tree is the daemon's new mainline: a checkpoint/rewind cycle -# round-trips it, and the old tree is retained for the trash TTL. -acy checkpoint --wait --kind post >/dev/null || fail "S9: checkpoint after apply" -echo "$OUT" | grep -q "reload your editor" || fail "S9: no editor warning: $OUT" - -# --- S10: apply conflicts legibly when the mainline moved ----------------- -acy session-start dry-4 --host acceptance >/dev/null || fail "S10: session-start" -printf 'FORK-EDIT\n' > "$R/src/main.rs" -acy session-resolve dry-4 >/dev/null || fail "S10: resolve" -printf 'MAINLINE-EDIT\n' > "$R/src/main.rs" -settle 0.4 -acy checkpoint --wait --durable --kind post >/dev/null || fail "S10: mainline checkpoint" -set +e -OUT="$(acy session-apply dry-4 2>&1)" -CODE=$? -set -e -[ "$CODE" -ne 0 ] || fail "S10: apply succeeded over a moved mainline: $OUT" -echo "$OUT" | grep -qi "conflict\|moved\|behind" || fail "S10: conflict not legible: $OUT" -[ "$(cat "$R/src/main.rs")" = "MAINLINE-EDIT" ] || fail "S10: conflicting apply touched the tree" - -# --- S11: guarded paths also hold on explicit forks ---------------------- -# Verified via server truth (promote), not client reads: the macOS NFS -# client caches a doomed write and would echo it back on `cat`, so a -# guarded write's refusal shows only in what promote actually lands. -FORK_OUT="$(acy fork -n 1)" || fail "S11: fork" -FID="$(echo "$FORK_OUT" | awk '/^fork /{print $2}')" -F="$(dirname "$R")/.$(basename "$R").forks/mnt/$FID" -[ -f "$F/.env" ] || fail "S11: fork missing .env" -try bash -c "printf 'LEAK\n' > '$F/.env'" # guarded — must not land -try rm "$F/migrations/001.sql" # guarded — must not land -printf 'S11-FORK\n' > "$F/src/main.rs" || fail "S11: fork unguarded write" -settle 0.4 -acy promote "$FID" >/dev/null || fail "S11: promote" -# Only the unguarded edit reached the real tree; guarded paths are intact. -[ "$(cat "$R/src/main.rs")" = "S11-FORK" ] || fail "S11: unguarded edit not landed" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S11: guarded .env landed a change" -[ "$(cat "$R/migrations/001.sql")" = "CREATE TABLE a;" ] || fail "S11: guarded migration changed" - -# --- S12: stop unmounts an active shadow; nothing is left on the root ----- -acy session-start dry-5 --host acceptance >/dev/null || fail "S12: session-start" -printf 'LOST\n' > "$R/src/main.rs" -shadowed || fail "S12: not shadowed" -acy stop >/dev/null || fail "S12: stop" -sleep 0.5 -shadowed && fail "S12: shadow left mounted after stop" -[ "$(cat "$R/src/main.rs")" = "S11-FORK" ] || fail "S12: stop leaked shadow writes: $(cat "$R/src/main.rs")" - -# --- S13: kill -9 mid-session: the next daemon sweeps the dead shadow ----- -acy status >/dev/null || fail "S13: respawn" -acy session-start dry-6 --host acceptance >/dev/null || fail "S13: session-start" -printf 'LOST-2\n' > "$R/src/main.rs" -shadowed || fail "S13: not shadowed" -PID="$(daemon_pid)" -[ -n "$PID" ] || fail "S13: no daemon pid" -kill -9 "$PID" -sleep 0.5 -# A dead NFS server behind the mount: the real tree must come back on the -# next daemon start, without a manual umount. -acy status >/dev/null || fail "S13: daemon restart with dead shadow" -sleep 0.3 -shadowed && fail "S13: dead shadow not swept" -[ "$(cat "$R/src/main.rs")" = "S11-FORK" ] || fail "S13: crash leaked shadow writes: $(cat "$R/src/main.rs")" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S13: .env after crash" -must_fail "S13: resolve dead session" acy session-resolve dry-6 -# And the repo is fully usable again: a new session works end to end. -acy session-start dry-7 --host acceptance >/dev/null || fail "S13: session after sweep" -printf 'AFTER-CRASH\n' > "$R/src/main.rs" -acy session-resolve dry-7 | grep -q "^M src/main.rs" || fail "S13: resolve after sweep" -acy session-apply dry-7 >/dev/null || fail "S13: apply after sweep" -[ "$(cat "$R/src/main.rs")" = "AFTER-CRASH" ] || fail "S13: apply after sweep" - -pass "safe-mode: shadow session, guarded paths vs shell, resolve/discard/apply, conflict, stop + crash sweep" From 79ec5827a3ceeba64fcbaf7ee23ac6798d48f947 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 01:54:09 +0100 Subject: [PATCH 027/106] Avoid duplicate daemon repository recovery --- crates/acyclic/src/main.rs | 3 +++ crates/acyclic/src/server.rs | 12 +++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index 6a46099..14ac84f 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -239,6 +239,9 @@ fn main() { libc::signal(libc::SIGPIPE, libc::SIG_DFL); } } + if matches!(cli.command, Command::Daemon { .. }) { + std::process::exit(run(cli, Path::new("."))); + } let repo_arg = cli.repo.clone().unwrap_or_else(|| PathBuf::from(".")); acyclic::fork::reap_legacy_shadow(&repo_arg); let repo_arg = acyclic::rewind::recover_before_repo_open(&repo_arg).unwrap_or_else(|error| { diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 0e21b5b..32ad516 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -157,12 +157,14 @@ pub fn run(repo_root: &Path) -> Result<(), String> { ); phase = std::time::Instant::now(); }; - rewind::recover_before_repo_open(repo_root).map_err(|error| error.to_string())?; + fork::reap_legacy_shadow(repo_root); + let repo_root = + rewind::recover_before_repo_open(repo_root).map_err(|error| error.to_string())?; lap("rewind recovery before repo open"); - let config = Config::load(repo_root).map_err(|error| error.to_string())?; + let config = Config::load(&repo_root).map_err(|error| error.to_string())?; lap("config load"); let stores_root = config.store_dir.as_ref().map(PathBuf::from); - let paths = StorePaths::for_repo(repo_root, stores_root.as_deref()) + let paths = StorePaths::for_repo(&repo_root, stores_root.as_deref()) .map_err(|error| error.to_string())?; // Finish or unwind any rewind that a crash interrupted BEFORE the store @@ -175,7 +177,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { ) .map_err(|error| error.to_string())?; lap("rewind journal recovery"); - fork::sweep_stale_forks(repo_root); + fork::sweep_stale_forks(&repo_root); lap("sweep stale forks"); // Same reason as the fork sweep, and the same moment: a model run a // crashed daemon left behind is still running, and still billing. @@ -191,7 +193,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { let listener = bind_socket(&runtime, &paths)?; lap("socket bind + pidfile"); let store = runtime - .block_on(Store::open(repo_root, paths.clone())) + .block_on(Store::open(&repo_root, paths.clone())) .map_err(|error| error.to_string())?; let repo_root = store.repo_root.clone(); lap("store open"); From ac48a3fae8ff2c63e856adae6398ab33ea6c6312 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 04:26:39 +0100 Subject: [PATCH 028/106] Require native mounts for plugin forks --- Cargo.lock | 13 ++ crates/acyclic/src/fork.rs | 103 ++-------------- crates/acyclic/src/pipeline.rs | 63 ---------- crates/acyclic/src/rewind.rs | 42 +------ crates/acyclic/src/server.rs | 212 ++++++--------------------------- 5 files changed, 57 insertions(+), 376 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae3face..59c8cc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,7 @@ dependencies = [ name = "acyclic-fs" version = "0.2.0-rc.5" dependencies = [ + "acyclic-native-runtime", "acyclic-objects", "acyclic-stream", "async-trait", @@ -57,10 +58,21 @@ dependencies = [ "windows", ] +[[package]] +name = "acyclic-native-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "io-uring", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "acyclic-objects" version = "1.0.0-rc.4" dependencies = [ + "acyclic-native-runtime", "async-trait", "blake3", "bytes", @@ -81,6 +93,7 @@ dependencies = [ name = "acyclic-stream" version = "1.0.0-rc.7" dependencies = [ + "acyclic-native-runtime", "async-trait", "bytes", "fs2", diff --git a/crates/acyclic/src/fork.rs b/crates/acyclic/src/fork.rs index ff79275..d5a26cb 100644 --- a/crates/acyclic/src/fork.rs +++ b/crates/acyclic/src/fork.rs @@ -7,45 +7,16 @@ //! the volume); the daemon owns the mount sessions (they must live in the //! long-lived process). //! -//! When the host has no mount provider (no usable `/dev/fuse` on Linux, or -//! loopback NFS blocked on macOS) a fork degrades to a *copy*: the base -//! generation is materialized into a real directory, and at promote time -//! that directory is captured back into the fork's overlay so the same -//! commit, conflict check, and swap run unchanged. The promise a fork makes -//! — a writable tree that never touches the real one until promoted — holds -//! either way; only the O(1) creation cost is lost. - use std::path::{Path, PathBuf}; use std::sync::Arc; use acyclic_fs::model::VolumeConfig; use acyclic_fs::SharedCheckout; use acyclic_fs::{ - capture_baseline, capture_root_identity, probe_native_mount, CancellationToken, CaptureOptions, - GenerationId, LocalAuthorityBackend, LocalObjectBackend, NativeMountKind, VolumeId, - WorkCounters, + probe_native_mount, GenerationId, LocalAuthorityBackend, LocalObjectBackend, NativeMountKind, + VolumeId, }; -use crate::{EngineError, Result}; - -/// How a fork is realized on this host. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ForkMode { - /// Routed native mount: O(1) creation, lazy hydration. - Mount, - /// Materialized directory: full copy up front, captured back at promote. - Copy, -} - -impl ForkMode { - pub fn as_str(self) -> &'static str { - match self { - ForkMode::Mount => "mount", - ForkMode::Copy => "copy", - } - } -} - /// What the host can do for fork mounts, probed once at daemon start and /// reported by `status`/`init`. #[derive(Clone, Debug)] @@ -57,16 +28,6 @@ pub struct MountCapability { pub reason: Option, } -impl MountCapability { - pub fn fork_mode(&self) -> ForkMode { - if self.available { - ForkMode::Mount - } else { - ForkMode::Copy - } - } -} - /// Live probe of the native mount provider. pub fn mount_capability() -> MountCapability { let probe = probe_native_mount(); @@ -89,55 +50,28 @@ pub fn mount_capability() -> MountCapability { pub fn mount_setup_hint() -> &'static str { if cfg!(target_os = "linux") { concat!( - "forks will use full copies until /dev/fuse is usable. To enable mounts:\n", + "forks require usable /dev/fuse. To enable mounts:\n", " sudo modprobe fuse # load the kernel module\n", " sudo usermod -aG fuse \"$USER\" # if /dev/fuse is group-restricted; log in again\n", " docker run --device /dev/fuse --cap-add SYS_ADMIN ... # inside a container", ) } else if cfg!(target_os = "macos") { concat!( - "forks will use full copies: the built-in NFS mount tools (/sbin/mount_nfs, /sbin/umount)\n", + "forks require the built-in NFS mount tools (/sbin/mount_nfs, /sbin/umount), which\n", "are missing or blocked by policy. No extra software is needed on macOS;\n", "ask your administrator to allow loopback NFS mounts.", ) } else if cfg!(windows) { concat!( - "forks will use full copies until the optional Windows Projected File System\n", - "feature is enabled and available to this process. Enable Client-ProjFS in\n", - "Windows Features to use accelerated forks. ProjFS safely rejects cross-root\n", + "forks require the optional Windows Projected File System feature. Enable\n", + "Client-ProjFS in Windows Features. ProjFS safely rejects cross-root\n", "directory moves that it cannot capture atomically.", ) } else { - "native mounts are not supported on this platform; forks use full copies." + "native mounts are required for forks and are not supported on this platform." } } -/// Root for copy-mode fork directories (a sibling of the mount root). -pub fn forks_copy_root(repo_root: &Path) -> Option { - Some(forks_root(repo_root)?.join("copy")) -} - -/// Captures the current content of `root` into a fork's overlay, so that -/// promote sees it exactly as it would see writes through a mount. -/// The overlay must be pristine (a fresh fork seed): capture is a full-tree -/// baseline against the checkout, so only paths that actually differ from -/// the base become pending mutations. -pub async fn capture_copy(shared: &SharedLocalCheckout, root: &Path) -> Result<()> { - let options = CaptureOptions { - source_root: root.to_path_buf(), - expected_root_identity: capture_root_identity(root) - .map_err(EngineError::fs("copy root identity"))?, - maximum_paths: 4_000_000, - maximum_extent_spans: 65_536, - }; - let cancel = CancellationToken::new(); - let mut guard = shared.lock().await; - capture_baseline(&mut guard, &options, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("capture copy fork"))?; - Ok(()) -} - /// The mount-safe checkout wrapper for the local backend. pub type SharedLocalCheckout = SharedCheckout; @@ -258,34 +192,13 @@ mod tests { use super::*; #[test] - fn probe_names_a_provider_and_picks_a_mode() { + fn probe_names_a_provider() { let capability = mount_capability(); assert_ne!(capability.provider, ""); assert_eq!(capability.available, capability.reason.is_none()); - assert_eq!( - capability.fork_mode(), - if capability.available { - ForkMode::Mount - } else { - ForkMode::Copy - } - ); assert!(!mount_setup_hint().is_empty()); } - #[test] - fn copy_root_sits_beside_mount_root() { - let repo = Path::new("/work/my-repo"); - assert_eq!( - forks_copy_root(repo).expect("copy"), - Path::new("/work/.my-repo.forks/copy") - ); - assert_eq!( - forks_mount_root(repo).expect("mnt"), - Path::new("/work/.my-repo.forks/mnt") - ); - } - #[test] fn forks_root_is_a_hidden_sibling() { let root = forks_root(Path::new("/work/my-repo")).expect("root"); diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index d954904..d1589ae 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -200,19 +200,6 @@ enum Request { paths: Vec, reply: oneshot::Sender>, }, - /// Restores one path from `target` into `root` (a copy fork's directory). - RestorePathInto { - target: GenerationId, - root: PathBuf, - path: PathBuf, - reply: oneshot::Sender>, - }, - /// Copy-mode fork: write `generation` out to `destination`. - Materialize { - generation: GenerationId, - destination: PathBuf, - reply: oneshot::Sender>, - }, Promote { shared: Arc, base: GenerationId, @@ -527,34 +514,6 @@ impl PipelineHandle { )? } - /// Restores one path from `target` into `root` rather than the working tree. - pub async fn restore_path_into( - &self, - target: GenerationId, - root: PathBuf, - path: PathBuf, - ) -> Result { - request!( - self, - RestorePathInto { - target: target, - root: root, - path: path - } - )? - } - - /// Copy-mode fork: materializes `generation` into `destination`. - pub async fn materialize(&self, generation: GenerationId, destination: PathBuf) -> Result<()> { - request!( - self, - Materialize { - generation: generation, - destination: destination - } - )? - } - pub async fn promote( &self, shared: Arc, @@ -792,14 +751,12 @@ fn fail_request(request: Request, message: &str) { Request::Diff { reply, .. } => drop(reply.send(Err(error()))), Request::Fork { reply } => drop(reply.send(Err(error()))), Request::Promote { reply, .. } => drop(reply.send(Err(error()))), - Request::Materialize { reply, .. } => drop(reply.send(Err(error()))), Request::ScratchCheckout { reply, .. } => drop(reply.send(Err(error()))), Request::PublishHead { reply } => drop(reply.send(Err(error()))), Request::MergePlan { reply, .. } => drop(reply.send(Err(error()))), Request::BuildGeneration { reply, .. } => drop(reply.send(Err(error()))), Request::ApplyToOverlay { reply, .. } => drop(reply.send(Err(error()))), Request::ReadFiles { reply, .. } => drop(reply.send(Err(error()))), - Request::RestorePathInto { reply, .. } => drop(reply.send(Err(error()))), Request::MaterializePaths { reply, .. } => drop(reply.send(Err(error()))), Request::RecordGeneration { reply, .. } => drop(reply.send(Err(error()))), Request::SnapshotOverlay { reply, .. } => drop(reply.send(Err(error()))), @@ -1188,26 +1145,6 @@ impl Pipeline { let _ = reply.send(result); false } - Request::RestorePathInto { - target, - root, - path, - reply, - } => { - let result = - Box::pin(rewind::restore_path_into(&self.store, target, &root, &path)).await; - let _ = reply.send(result); - false - } - Request::Materialize { - generation, - destination, - reply, - } => { - let _ = reply - .send(rewind::materialize_into(&self.store, generation, &destination).await); - false - } Request::Promote { shared, base, diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index e46ddfa..61ae7e5 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -61,19 +61,8 @@ pub async fn restore_path( relative: &Path, ) -> Result { let root = store.repo_root.clone(); - restore_path_into(store, target, &root, relative).await -} - -/// [`restore_path`] against an arbitrary root directory instead of the -/// working tree: a copy-mode fork's directory during a rebase. -pub async fn restore_path_into( - store: &Store, - target: GenerationId, - root: &Path, - relative: &Path, -) -> Result { let mut checkout = store.checkout_exact(target).await?; - materialize_path_into_checkout(&mut checkout, root, relative, PathReplace::Atomic).await + materialize_path_into_checkout(&mut checkout, &root, relative, PathReplace::Atomic).await } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -499,35 +488,6 @@ pub fn recover_before_repo_open(repo_root: &Path) -> Result { Ok(canonical) } -/// Materializes `generation` into `destination`, which must not exist yet. -/// Used for copy-mode forks; a rewind does the same into its temp sibling. -pub async fn materialize_into( - store: &Store, - generation: GenerationId, - destination: &Path, -) -> Result<()> { - std::fs::create_dir(destination)?; - let mut checkout = store.checkout_exact(generation).await?; - let cancel = CancellationToken::new(); - materialize_checkout( - &mut checkout, - &MaterializeOptions { - destination: destination.to_path_buf(), - maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, - maximum_extent_spans: MAXIMUM_EXTENT_SPANS, - transfer_bytes: TRANSFER_BYTES, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(|error| { - let _ = std::fs::remove_dir_all(destination); - EngineError::Restore(format!("materialize: {error:?}")) - })?; - Ok(()) -} - /// Executes a rewind against the store's repo. Called from the pipeline with /// captures paused; the caller re-baselines afterwards. Excluded paths are /// carried from the live tree into the restored one: no checkpoint holds diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 32ad516..8554492 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant}; use crate::ipc; use crate::proto; use acyclic::config::Config; -use acyclic::fork::{self, ForkMode, MountCapability, SharedLocalCheckout}; +use acyclic::fork::{self, MountCapability, SharedLocalCheckout}; use acyclic::guard::GuardedMountFilesystem; use acyclic::index::{Attribution, CheckpointKind, CheckpointRow, Index}; use acyclic::merge::{self, Entry}; @@ -36,9 +36,6 @@ struct ForkState { /// head it was rebased onto. base: acyclic::GenerationId, entry: proto::ForkEntry, - /// Copy-mode forks only: the materialized directory the user works in. - /// Captured back into `shared` at promote, removed at drop/promote. - copy_dir: Option, /// Set by a conflicting promote until the markers are gone. conflict: Option, } @@ -203,16 +200,11 @@ pub fn run(repo_root: &Path) -> Result<(), String> { lap("pipeline thread spawn (baseline runs on it)"); let shutdown = Arc::new(Notify::new()); - let mut mounts = fork::mount_capability(); + let mounts = fork::mount_capability(); lap("native mount probe"); - // Test hook: exercise the copy-fork paths on a host that has mounts. - if std::env::var_os("ACYCLIC_FORCE_COPY_FORKS").is_some() { - mounts.available = false; - mounts.reason = Some("ACYCLIC_FORCE_COPY_FORKS is set".into()); - } if !mounts.available { eprintln!( - "{NAME} daemon: mounts unavailable ({}): forks fall back to copies", + "{NAME} daemon: mounts unavailable ({}): forks are disabled", mounts.reason.as_deref().unwrap_or("unknown reason") ); } @@ -702,10 +694,8 @@ impl Server { .map(|(id, _)| id.clone()) .collect(); for id in scratch_ids { - let fork = self.forks.lock().await.remove(&id); - let copy_dir = fork.and_then(|fork| fork.copy_dir); - if let Err(error) = self.discard_fork_workspace(&id, copy_dir.as_deref()).await - { + self.forks.lock().await.remove(&id); + if let Err(error) = self.detach_route(&id).await { eprintln!("{NAME} daemon: drop scratch fork {id}: {error}"); } } @@ -750,8 +740,13 @@ impl Server { if count == 0 || count > 16 { return Err("fork count must be 1..=16".into()); } - if self.mounts.fork_mode() == ForkMode::Copy { - return self.fork_copies(count, session_id).await; + if !self.mounts.available { + return Err(format!( + "native {} mounts are unavailable: {}\n{}", + self.mounts.provider, + self.mounts.reason.as_deref().unwrap_or("unknown reason"), + fork::mount_setup_hint() + )); } let root = fork::forks_mount_root(&self.repo_root) .ok_or("repo root has no parent for fork workspaces")?; @@ -764,7 +759,7 @@ impl Server { let entry = proto::ForkEntry { id: id.clone(), path: root.join(&id).display().to_string(), - mode: ForkMode::Mount.as_str().to_owned(), + mode: "mount".to_owned(), base: acyclic::generation_hex(seed.base), created_at: unix_now(), session_id: session_id.clone(), @@ -777,7 +772,6 @@ impl Server { shared: seed.shared, base: seed.base, entry: clone_entry(&entry), - copy_dir: None, conflict: None, }, ); @@ -796,10 +790,9 @@ impl Server { } proto::Op::ForkDrop { id } => { let mut forks = self.forks.lock().await; - let fork = forks.remove(&id).ok_or(format!("no fork {id}"))?; + forks.remove(&id).ok_or(format!("no fork {id}"))?; drop(forks); - self.discard_fork_workspace(&id, fork.copy_dir.as_deref()) - .await?; + self.detach_route(&id).await?; Ok(proto::Reply::Unit) } proto::Op::Promote { id } => { @@ -861,12 +854,8 @@ impl Server { } Ok(landed) => landed, }; - // Landed: the fork is consumed. Drop its route (mount fork) - // or its directory (copy fork). - if let Err(error) = self - .discard_fork_workspace(&id, fork.copy_dir.as_deref()) - .await - { + // Landed: the fork is consumed. Drop its mount route. + if let Err(error) = self.detach_route(&id).await { eprintln!("{NAME} daemon: discard fork {id} after promote: {error}"); } match landed { @@ -909,12 +898,11 @@ impl Server { } } proto::Op::ForkDiff { id } => { - let (base, shared, capture_dir) = { + let (base, overlay) = { let forks = self.forks.lock().await; let fork = forks.get(&id).ok_or(format!("no fork {id}"))?; - (fork.base, Arc::clone(&fork.shared), fork.copy_dir.clone()) + (fork.base, Arc::clone(&fork.shared)) }; - let overlay = self.fork_view(base, shared, capture_dir.as_deref()).await?; let changes = if overlay.lock().await.has_pending_mutations() { let generation = self .handle @@ -928,8 +916,6 @@ impl Server { } else { Vec::new() }; - // Timestamps differ on a copy fork by construction; only - // content is a blast radius. Ok(proto::Reply::Diff( self.annotate_ignored( changes @@ -945,79 +931,6 @@ impl Server { } } - /// Copy-mode forks: materialize the base generation into a real - /// directory per fork. Same ids, lifecycle, and promote semantics as - /// mounted forks; creation is O(tree) instead of O(1). - async fn fork_copies( - &self, - count: u32, - session_id: Option, - ) -> Result { - let root = fork::forks_copy_root(&self.repo_root) - .ok_or("repo root has no parent for fork workspaces")?; - std::fs::create_dir_all(&root).map_err(|error| error.to_string())?; - let mut created = Vec::new(); - for _ in 0..count { - let seed = self.handle.fork().await.map_err(stringify)?; - let id = short_id(); - let dir = root.join(&id); - self.handle - .materialize(seed.base, dir.clone()) - .await - .map_err(stringify)?; - let entry = proto::ForkEntry { - id: id.clone(), - path: dir.display().to_string(), - mode: ForkMode::Copy.as_str().to_owned(), - base: acyclic::generation_hex(seed.base), - created_at: unix_now(), - session_id: session_id.clone(), - conflict_paths: Vec::new(), - conflict: None, - }; - self.forks.lock().await.insert( - id, - ForkState { - shared: seed.shared, - base: seed.base, - entry: clone_entry(&entry), - copy_dir: Some(dir), - conflict: None, - }, - ); - created.push(entry); - } - Ok(proto::Reply::Forks(created)) - } - - /// Uses the live overlay for mounted forks. Copy forks are reconciled - /// from the host tree into a scratch checkout before diff or promotion. - async fn fork_view( - &self, - base: acyclic::GenerationId, - shared: Arc, - capture_dir: Option<&Path>, - ) -> Result, String> { - let Some(dir) = capture_dir else { - return Ok(shared); - }; - let started = std::time::Instant::now(); - let scratch = self - .handle - .scratch_checkout(base) - .await - .map_err(stringify)?; - let scratch_ms = acyclic::trace::ms(started); - let capture_started = std::time::Instant::now(); - fork::capture_copy(&scratch, dir).await.map_err(stringify)?; - acyclic::trace!( - "daemon", - "copy fork capture: scratch {scratch_ms:.1}ms, full-tree capture {:.1}ms", - acyclic::trace::ms(capture_started) - ); - Ok(scratch) - } - /// Lands a fork in place. The fork's paths are merged onto the current /// head (a three-way merge when the mainline moved; a plain write of /// the fork's paths when it did not) and written onto the real tree @@ -1030,10 +943,7 @@ impl Server { fork: &ForkState, label: &str, ) -> Result { - let capture_dir = fork.copy_dir.as_deref(); - let overlay = self - .fork_view(fork.base, Arc::clone(&fork.shared), capture_dir) - .await?; + let overlay = Arc::clone(&fork.shared); // A rebased fork must have resolved its markers before it can land. if let Some(conflict) = fork.conflict.as_ref() { self.refuse_unresolved_markers(&overlay, conflict).await?; @@ -1042,13 +952,8 @@ impl Server { let head = self.handle.publish_head().await.map_err(stringify)?; acyclic::trace!( "daemon", - "promote {id}: publish_head {:.1}ms; {} fork, mainline {} since the fork's base", + "promote {id}: publish_head {:.1}ms; mainline {} since the fork's base", acyclic::trace::ms(publish_started), - if fork.copy_dir.is_some() { - "copy" - } else { - "mount" - }, if head == fork.base { "UNMOVED (plain in-place write)" } else { @@ -1056,14 +961,7 @@ impl Server { } ); let landed = self - .merge_onto_head( - id, - overlay, - fork.base, - head, - fork.copy_dir.as_deref(), - label, - ) + .merge_onto_head(id, overlay, fork.base, head, label) .await; acyclic::trace!( "daemon", @@ -1227,7 +1125,6 @@ impl Server { overlay: Arc, base: acyclic::GenerationId, head: acyclic::GenerationId, - copy_dir: Option<&Path>, label: &str, ) -> Result { let moved = head != base; @@ -1398,7 +1295,7 @@ impl Server { ) .await .map_err(stringify)?; - self.rebase_fork(id, snapshot, rebased, copy_dir).await?; + self.rebase_fork(id, snapshot, rebased).await?; let mut files: Vec = plan .conflicted .iter() @@ -1495,15 +1392,13 @@ impl Server { }) } - /// Makes the fork workspace equal to `rebased`: writes R − F into the - /// fork's directory, through the mount for a mounted fork. The real - /// tree is never touched. + /// Makes the mounted fork equal to `rebased` by writing R − F through + /// the mount. The real tree is never touched. async fn rebase_fork( &self, id: &str, snapshot: acyclic::GenerationId, rebased: acyclic::GenerationId, - copy_dir: Option<&Path>, ) -> Result<(), String> { let changed: Vec = content_changes( self.handle @@ -1515,53 +1410,16 @@ impl Server { .map(|change| change.path) .collect(); let roots = merge::subtree_roots(&changed); - if let Some(dir) = copy_dir { - for root in &roots { - self.handle - .restore_path_into(rebased, dir.to_path_buf(), root.clone()) - .await - .map_err(stringify)?; - } - } else { - // A mounted fork: write THROUGH the mount, never behind it. - // The driver and the kernel keep name and attribute caches - // that only their own operations update; a write via the - // checkout leaves a file the fork had deleted invisible for - // good, and the FUSE transport has no invalidation at all. - let dir = fork::forks_mount_root(&self.repo_root) - .ok_or("repo root has no parent for fork workspaces")? - .join(id); - self.handle - .materialize_paths(rebased, dir, roots) - .await - .map_err(stringify)?; - } - Ok(()) - } - - /// Drops whatever backs a fork: its route for mounted forks, its - /// directory for copy forks. - async fn discard_fork_workspace( - &self, - id: &str, - copy_dir: Option<&Path>, - ) -> Result<(), String> { - match copy_dir { - Some(dir) => { - std::fs::remove_dir_all(dir) - .map_err(|error| format!("remove fork copy: {error}"))?; - Self::remove_if_empty(dir.parent()); - Ok(()) - } - None => self.detach_route(id).await, - } - } - - fn remove_if_empty(dir: Option<&Path>) { - if let Some(dir) = dir { - let _ = std::fs::remove_dir(dir); - let _ = dir.parent().map(std::fs::remove_dir); - } + // Write THROUGH the mount, never behind it. The driver and kernel + // keep name and attribute caches that only their own operations + // update. + let dir = fork::forks_mount_root(&self.repo_root) + .ok_or("repo root has no parent for fork workspaces")? + .join(id); + self.handle + .materialize_paths(rebased, dir, roots) + .await + .map_err(stringify) } /// Removes one fork's route; the session unmounts (and the mount root From 6a5827aecbb867253e95e6faeaff3de39a227b67 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 04:28:23 +0100 Subject: [PATCH 029/106] Remove obsolete plugin compatibility paths --- crates/acyclic/src/fork.rs | 37 ----------------------------- crates/acyclic/src/main.rs | 11 --------- crates/acyclic/src/server.rs | 1 - docs/design/00-overview.md | 2 +- docs/design/03-forks.md | 9 +++---- docs/design/implementation-forks.md | 2 +- docs/design/implementation-merge.md | 30 +++++++++-------------- docs/windows-verification.md | 2 +- 8 files changed, 17 insertions(+), 77 deletions(-) diff --git a/crates/acyclic/src/fork.rs b/crates/acyclic/src/fork.rs index d5a26cb..5d11e56 100644 --- a/crates/acyclic/src/fork.rs +++ b/crates/acyclic/src/fork.rs @@ -150,43 +150,6 @@ pub fn sweep_stale_forks(repo_root: &Path) { let _ = std::fs::remove_dir(&root); } -/// Recovers a repository still covered by a shadow mount from a legacy -/// `dry_run` daemon. New versions no longer create these mounts, but upgrade -/// must remain able to expose the real repository before opening it. -pub fn reap_legacy_shadow(repo: &Path) { - #[cfg(any(target_os = "macos", target_os = "linux"))] - { - use std::time::Duration; - let target = match (repo.parent(), repo.file_name()) { - (Some(parent), Some(name)) => parent - .canonicalize() - .map_or_else(|_| repo.to_path_buf(), |parent| parent.join(name)), - _ => repo.to_path_buf(), - }; - let probe = target.clone(); - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || drop(sender.send(std::fs::metadata(probe).is_ok()))); - if !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok(true)) { - #[cfg(target_os = "macos")] - drop( - std::process::Command::new("umount") - .arg("-f") - .arg(target) - .status(), - ); - #[cfg(target_os = "linux")] - drop( - std::process::Command::new("fusermount") - .arg("-u") - .arg(target) - .status(), - ); - } - } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - let _ = repo; -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index 14ac84f..a18228c 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -61,9 +61,6 @@ enum Command { /// a queued snapshot can include edits made before it runs. #[arg(long, conflicts_with = "durable")] no_wait: bool, - /// Accepted for compatibility: waiting is now the default. - #[arg(long, hide = true)] - wait: bool, /// Also publish to the durable authority (coarse boundary). #[arg(long)] durable: bool, @@ -243,7 +240,6 @@ fn main() { std::process::exit(run(cli, Path::new("."))); } let repo_arg = cli.repo.clone().unwrap_or_else(|| PathBuf::from(".")); - acyclic::fork::reap_legacy_shadow(&repo_arg); let repo_arg = acyclic::rewind::recover_before_repo_open(&repo_arg).unwrap_or_else(|error| { eprintln!("{}: rewind recovery: {error}", product::NAME); std::process::exit(1); @@ -580,7 +576,6 @@ fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), Str Command::Checkpoint { message, no_wait, - wait: _, durable, kind, session_id, @@ -878,12 +873,6 @@ fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), Str "{} fork(s) ready — work in them freely; `{NAME} promote ` keeps a winner", entries.len() ); - if entries.iter().any(|entry| entry.mode == "copy") { - println!( - "note: no mount provider on this host, so these are full copies \ - (`{NAME} status` explains; promote works the same)" - ); - } Ok(()) } Command::Forks => { diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 8554492..4e05b29 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -154,7 +154,6 @@ pub fn run(repo_root: &Path) -> Result<(), String> { ); phase = std::time::Instant::now(); }; - fork::reap_legacy_shadow(repo_root); let repo_root = rewind::recover_before_repo_open(repo_root).map_err(|error| error.to_string())?; lap("rewind recovery before repo open"); diff --git a/docs/design/00-overview.md b/docs/design/00-overview.md index 585ff47..e9a3b0e 100644 --- a/docs/design/00-overview.md +++ b/docs/design/00-overview.md @@ -48,7 +48,7 @@ Second load-bearing constraint, from compliance: **purge-through-history and sna ## Settled since this was written -1. **Fork engine mechanism** — decided the opposite way to the lean recorded here. Mounts shipped; there is no reflink or `clonefile` path in the codebase, and the fallback when no mount provider is available is a **full copy**, not a reflink. Forks are routes inside one kernel mount rather than N mounts. See `implementation-forks.md`. +1. **Fork engine mechanism** — mounts shipped. Forks require the native provider and are routes inside one kernel mount rather than N mounts. See `implementation-forks.md`. 2. **Checkpoint alignment in hosts without lifecycle hooks** — resolved by the MCP adapter plus the `auto_checkpoint_idle_ms` timer. MCP is a second adapter shape this doc's thesis line does not yet mention. ## Open questions (not yet settled) diff --git a/docs/design/03-forks.md b/docs/design/03-forks.md index 5ccd9f6..73f5624 100644 --- a/docs/design/03-forks.md +++ b/docs/design/03-forks.md @@ -23,12 +23,9 @@ The fork engine. The headline demo launch. This section previously read as a forward plan. Launch 3 shipped; what follows is what exists, including where it diverged from the plan. -- **Copy-on-write materialization** — decided as **mounts**, the opposite of the - lean this doc recorded. There is no reflink or `clonefile` path in the - codebase. All forks are routes inside **one** kernel mount, not N mounts. - Without a mount provider every fork is a **full copy**, so the O(1) claim holds - in mount mode only — the degraded path is keyed on mount availability, not on - filesystem reflink support. +- **Copy-on-write materialization** — implemented as native mounts. There is no + reflink, `clonefile`, or full-copy path. All forks are routes inside one + kernel mount, and a supported native mount provider is required. - **Fork workspace management** — partially. Teardown and placement exist; per-fork port allocation and env provisioning were dropped as out of scope. - **Subagent orchestration wiring** — shipped, but as a prompt-level skill and a diff --git a/docs/design/implementation-forks.md b/docs/design/implementation-forks.md index 8c9b7b8..233c08c 100644 --- a/docs/design/implementation-forks.md +++ b/docs/design/implementation-forks.md @@ -4,7 +4,7 @@ Ships `03-forks.md`: N-way local forks, pick the winner. Built on fs's native mounts — the resolution of the plan's open CoW question is **mounts, not reflinks**: fs now ships qualified FUSE-T/FUSE/ProjFS drivers and the mount path is the strategic asset (lazy hydration and filesystem-level guarded-path -enforcement). Reflinks remain a fallback if mount UX disappoints. +enforcement). A native mount provider is required. ## How a fork works (the fs wiring, verified against fsd's usage) diff --git a/docs/design/implementation-merge.md b/docs/design/implementation-merge.md index c8c1630..61979db 100644 --- a/docs/design/implementation-merge.md +++ b/docs/design/implementation-merge.md @@ -131,10 +131,8 @@ When at least one file is `Conflicted` and nothing is refused: 1. Build **R** = M with the conflicted files replaced by their marker-bearing content. R is F rebased onto H. -2. Write R's differences from F into the fork: for a mount fork, through - the fork's `SharedLocalCheckout` (the mount serves it live); for a - copy fork, into the copy directory as well. `capture_copy` already - handles copy → overlay at promote. +2. Write R's differences from F through the mounted workspace so the mount + and kernel caches observe every change. 3. Set `fork.base = H`. Record R as `fork rebased onto (N conflict(s))`. 4. Record the open conflict on the fork: `{ base: B, ours: F, theirs: H, @@ -309,10 +307,9 @@ Acceptance (`merge.sh`, each from a fresh fork set): ## Risks and open questions -- **Writing into a live mount fork.** R−F goes through the fork's shared - checkout while the agent may hold the mount open. Serialize under the - checkout lock like `capture_copy` does; document that an editor with - the file open sees the markers on next read. +- **Writing into a live mount fork.** R−F goes through the mounted path while + the agent may hold it open. The driver serializes the resulting overlay + mutations; an editor with the file open sees markers on its next read. - **Fork base mutation.** `ForkSeed.base` is immutable today and the daemon's fork table copies it. Both must update atomically with the rebase record; a daemon restart already loses forks, so no persistence @@ -376,13 +373,10 @@ Deliberate divergences: drain), recorded as the single landed row. The `before promote …` row records the published head without another drain. R is built from M with the marker files on top. A rebase writes R − F into the - fork: through the shared checkout for a mount fork (route detached - during promote, re-attached after), via `restore_path_into` for a copy - fork. -- **Rebased forks land through the copy-fork snapshot/apply path.** A rebased - mount fork's checkout still sits on the head it was cut from, so an optimistic - commit against the new head would conflict. `ForkState.rebased` routes - such forks through the same snapshot/apply path copy forks use. + fork through its mounted path so driver and kernel caches remain coherent. +- **Rebased forks land through snapshot/apply.** A rebased fork's checkout + still sits on the head it was cut from, so an optimistic commit against the + new head would conflict. - **Subtree copy and removal are iterative.** The fs facade's futures are large enough that three nested levels overflowed the pipeline thread's 2 MiB stack in a debug build. Both walks use an explicit work @@ -390,8 +384,6 @@ Deliberate divergences: - **`rewind --last` is not the undo.** It targets the latest real checkpoint, which after a merge is the landed row. The undo is the `before promote (merge)` safety row, exactly as for a replay. -- **`ACYCLIC_FORCE_COPY_FORKS`** makes a mount-capable host use copy - forks so `merge.sh` runs both modes on one machine. - **Wire.** `PromoteInfo` gained `merged_files`, `conflicts`, and `fork_path`; `ForkEntry` gained `conflict_paths` and `conflict` (base/ours/theirs). A conflicting promote returns a non-zero exit @@ -433,8 +425,8 @@ Deliberate divergences: `invalidate` returned Ok, and the FUSE transport (Linux, FUSE-T on the macOS runner) has no invalidation at all. The daemon now writes the rebase into `//…` with plain filesystem operations - (`merge::materialize_paths`), the same way copy forks get theirs, so - every cache saw the operation. Promote also no longer detaches the + (`merge::materialize_paths`), so every cache saw the operation. Promote + also no longer detaches the route before working; it snapshots under the checkout lock and drops the route only after the fork lands, which removed the negative-entry window that hid re-attached forks on Linux. diff --git a/docs/windows-verification.md b/docs/windows-verification.md index e00b591..a7f6581 100644 --- a/docs/windows-verification.md +++ b/docs/windows-verification.md @@ -22,7 +22,7 @@ lint job cannot see. | `crates/acyclic-engine/src/rewind.rs` | Windows directory exchange; journal write fixed; staging is retry-safe. | | `crates/acyclic-engine/src/guard.rs` | Guarded prefixes and `AppleDouble` matching in the host encoding. | | `crates/acyclic-engine/src/{diff,exclude}.rs` | Name decoding via `names` instead of per-file UTF-8 fallbacks. | -| `crates/acyclic-engine/src/fork.rs` | Windows is held to copy forks (see below). | +| `crates/acyclic/src/fork.rs` | Windows forks require Projected File System. | | `crates/acyclic/src/main.rs` | The client steps out of the tree before asking for a whole-tree swap. | | `.github/workflows/{ci,release}.yml` | A `windows` CI job; `win32/x64` release matrix entry. | | `packaging/npm/*.sh`, `deny.toml` | `win32` platform package; MSVC target in the license set. | From d66a827324ae18d56d82e572d5fbe1889915a972 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 04:44:47 +0100 Subject: [PATCH 030/106] Remove legacy Codex hook migration --- crates/acyclic/src/install.rs | 57 ----------------------------------- 1 file changed, 57 deletions(-) diff --git a/crates/acyclic/src/install.rs b/crates/acyclic/src/install.rs index 54d8b93..ad39f02 100644 --- a/crates/acyclic/src/install.rs +++ b/crates/acyclic/src/install.rs @@ -220,9 +220,6 @@ fn merge_hooks(path: &Path, host: &str) -> Result<(), String> { let root_map = root .as_object_mut() .ok_or_else(|| format!("{} is not an object", path.display()))?; - if host == "codex" { - remove_flat_codex_hooks(root_map); - } let hooks = root_map.entry("hooks").or_insert(json!({})); let hooks = hooks.as_object_mut().ok_or("hooks is not an object")?; merge_event_hooks(hooks, "hooks", host)?; @@ -232,22 +229,6 @@ fn merge_hooks(path: &Path, host: &str) -> Result<(), String> { Ok(()) } -/// Earlier releases wrote Codex's events at the top level of `hooks.json` -/// (`{"PreToolUse": [...]}`), a shape Codex 0.154 silently ignores. Drop -/// our entries from that layout so a re-install moves them under `hooks`; -/// anything a user put there is left alone. -fn remove_flat_codex_hooks(root: &mut serde_json::Map) { - for (event, _, _) in hook_events("codex") { - let Some(entries) = root.get_mut(event).and_then(Value::as_array_mut) else { - continue; - }; - entries.retain(|entry| !is_ours(entry)); - if entries.is_empty() { - root.remove(event); - } - } -} - /// Merges our entries into a map keyed by event name (the value under /// `"hooks"`). Same idempotency contract as `merge_hooks`. fn merge_event_hooks( @@ -1578,44 +1559,6 @@ mod tests { ); } - #[test] - fn codex_reinstall_migrates_the_legacy_flat_layout() { - let dir = tempfile::tempdir().expect("tempdir"); - let hooks_path = dir.path().join(".codex/hooks.json"); - std::fs::create_dir_all(dir.path().join(".codex")).expect("mkdir"); - let legacy = json!({ - "PreToolUse": [ - { "hooks": [{ "type": "command", "command": "echo user-hook" }] }, - { "hooks": [{ "type": "command", - "command": format!("ACYCLIC_HOST=codex {NAME} hook pre-tool") }] } - ], - "SessionEnd": [ - { "hooks": [{ "type": "command", - "command": format!("ACYCLIC_HOST=codex {NAME} hook session-end") }] } - ] - }); - std::fs::write(&hooks_path, legacy.to_string()).expect("seed"); - - codex(dir.path()).expect("install"); - let value: Value = - serde_json::from_str(&std::fs::read_to_string(&hooks_path).expect("read")) - .expect("json"); - assert_eq!( - value["PreToolUse"].as_array().map(Vec::len), - Some(1), - "user hook kept" - ); - assert!( - value.get("SessionEnd").is_none(), - "emptied legacy key removed" - ); - assert_eq!( - value["hooks"]["PreToolUse"].as_array().map(Vec::len), - Some(1), - "ours lives under hooks now" - ); - } - #[test] fn cursor_install_is_idempotent_and_writes_rule() { let dir = tempfile::tempdir().expect("tempdir"); From 8a48708926f3dd51f02741f9299ae53711b0b02e Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 05:03:21 +0100 Subject: [PATCH 031/106] Remove plugin recovery fallback paths --- crates/acyclic/src/install.rs | 10 +-- crates/acyclic/src/main.rs | 2 +- crates/acyclic/src/rewind.rs | 120 ++++++++++++------------------ crates/acyclic/src/server.rs | 7 +- tests/acceptance/cursor-e2e.sh | 4 +- tests/acceptance/windows-smoke.sh | 2 +- 6 files changed, 59 insertions(+), 86 deletions(-) diff --git a/crates/acyclic/src/install.rs b/crates/acyclic/src/install.rs index ad39f02..027c8de 100644 --- a/crates/acyclic/src/install.rs +++ b/crates/acyclic/src/install.rs @@ -8,8 +8,8 @@ //! API, so they get `acyclic mcp` (see `crate::mcp`) registered as an MCP //! server in whatever config file that host reads — project-scoped and //! checked in where the host supports it, the user's global config where -//! it doesn't. agents-md is the fallback for anything shell-capable: a -//! cheatsheet block in AGENTS.md and no hooks at all. +//! it doesn't. The agents-md adapter gives shell-capable hosts a cheatsheet +//! block in AGENTS.md and no hooks. //! //! Each `HostAdapter` below documents exactly what its host gets. The //! README's per-host table is the user-facing version of the same list, @@ -289,7 +289,7 @@ fn is_our_command(command: &str) -> bool { /// `~/.codex/config.toml` / `.codex/config.toml`. If that also means they /// share whatever fires `.codex/hooks.json`'s lifecycle events, this /// adapter may already cover the desktop app and IDE extension too, with no -/// new code — verify that first. The MCP fallback is already known to +/// new code — verify that first. MCP is already known to /// work (docs/manual-testing.md): `[mcp_servers.]` with `command`, /// `args` and `default_tools_approval_mode = "approve"` (without it a /// non-interactive session rejects every call). It is TOML, so a writer @@ -312,7 +312,7 @@ fn codex(repo: &Path) -> Result<(), String> { /// and `type` directly rather than Claude/Codex's nested `hooks` array), /// plus an always-applied project rule so the model has the CLI verbs in /// context. Cursor's hook events and payload shape differ from Claude -/// Code/Codex (see `hook::Payload`'s `conversation_id`/`command` fallbacks +/// Code/Codex (see `hook::Payload`'s host-specific fields /// and `hook::run`'s Cursor-only `{"permission":"allow"}` reply), so this /// writes Cursor's own event names rather than reusing `hook_events()`. fn cursor(repo: &Path) -> Result<(), String> { @@ -1283,7 +1283,7 @@ After the final round: `{{name}} diff ` and summarise the blast radius, ignoring `m` (metadata-only) lines. Mention anything a script or generator wrote. -## 6. Failure and fallback +## 6. Failure handling - A promote that reports `N file(s) conflict` has written conflict markers into THAT FORK (the mainline is untouched) and moved the fork diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index a18228c..19ce54b 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -265,7 +265,7 @@ fn main() { /// True when `repo` (canonical) lies inside a store's trash (an ancestor /// named `trash` whose parent is a store root, marked by `meta.json`), or -/// inside a rewind's sibling fallback trash directory (`..acyclic-trash-*`). +/// inside a rewind's sibling trash directory (`..acyclic-trash-*`). fn stranded_in_trash(repo: &Path) -> bool { repo.ancestors().any(|ancestor| { let Some(name) = ancestor.file_name().map(|name| name.to_string_lossy()) else { diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 61ae7e5..f27c756 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -1,6 +1,6 @@ //! Full-tree rewind: materialize the target generation into a sibling temp //! directory, atomically exchange it with the working tree, keep the old tree -//! in trash, and journal every phase so kill -9 leaves the repo fully-old or +//! beside the repository, and journal every phase so kill -9 leaves the repo fully-old or //! fully-new — never mixed. use std::path::{Component, Path, PathBuf}; @@ -26,7 +26,7 @@ static PARK_SEQUENCE: AtomicU64 = AtomicU64::new(0); #[derive(Clone, Debug)] pub struct RewindOutcome { pub restored: GenerationId, - /// Where the replaced tree went (trash, TTL-pruned). + /// Where the replaced tree was retained beside the repository. pub old_tree: PathBuf, /// User-facing caveat: open editors keep inodes from the old tree. pub warning: &'static str, @@ -403,7 +403,7 @@ pub enum Phase { pub struct RecoveredSwap { /// The target was already published when a Windows parking rename failed. pub published: bool, - /// The displaced tree retained in trash or a sibling location. + /// The displaced tree retained beside the repository. pub old_tree: Option, } @@ -414,8 +414,6 @@ pub struct RecoveredSwap { struct RecoveryLocator { repo_root: PathBuf, journal: PathBuf, - trash: PathBuf, - trash_ttl_days: u32, } fn locator_path(repo_root: &Path) -> Result { @@ -483,7 +481,7 @@ pub fn recover_before_repo_open(repo_root: &Path) -> Result { if locator.repo_root != canonical { return Err(EngineError::Restore("rewind locator repo mismatch".into())); } - recover(&locator.journal, &locator.trash, locator.trash_ttl_days)?; + recover(&locator.journal)?; std::fs::remove_file(locator_path)?; Ok(canonical) } @@ -510,11 +508,10 @@ pub async fn execute( let nonce = std::process::id(); let tmp = parent.join(format!(".{name}.{}-tmp-{nonce}", crate::product::NAME)); let journal_path = store.paths.rewind_journal(); - let trash_root = store.paths.trash(); // A failed previous exchange may have left a complete tree in scratch. // Resolve its journal before reusing either the temporary name or journal. - recover(&journal_path, &trash_root, trash_ttl_days)?; + recover(&journal_path)?; // 1. Materialize the target into an empty sibling directory. A tmp left // by an earlier attempt that failed before the swap is stale by @@ -593,24 +590,18 @@ pub async fn execute( carried, }, )?; - let locator = publish_locator(repo, &journal_path, &trash_root, trash_ttl_days)?; + let locator = publish_locator(repo, &journal_path)?; if let Err(error) = atomic_exchange(repo, &tmp) { // The third Windows rename can fail after the new tree is already // published. Reconcile immediately, before the daemon accepts another // rewind that could reuse the scratch name. - return reconcile_exchange_failure( - target, - repo, - &journal_path, - &trash_root, - trash_ttl_days, - error, - ); + return reconcile_exchange_failure(target, repo, &journal_path, error); } - // 4. Old tree to trash (best effort: EXDEV falls back to a sibling path). - let old_tree = park_replaced_tree(&tmp, &trash_root, parent, &name)?; - finish_published_rewind(&journal_path, &locator, &trash_root, trash_ttl_days); + // 4. Retain the old tree beside the repository. + let old_tree = park_replaced_tree(&tmp, parent, &name)?; + finish_published_rewind(&journal_path, &locator); + prune_sibling_trash(repo, trash_ttl_days); Ok(RewindOutcome { restored: target, @@ -619,24 +610,21 @@ pub async fn execute( }) } -fn finish_published_rewind(journal: &Path, locator: &Path, trash: &Path, ttl_days: u32) { +fn finish_published_rewind(journal: &Path, locator: &Path) { // The tree is already published. Cleanup failure leaves the journal and // locator for startup retry, but must not report a failed rewind. if std::fs::remove_file(journal).is_ok() { let _ = std::fs::remove_file(locator); } - prune_trash(trash, ttl_days); } fn reconcile_exchange_failure( target: GenerationId, repo: &Path, journal_path: &Path, - trash_root: &Path, - trash_ttl_days: u32, error: EngineError, ) -> Result { - let recovered = recover(journal_path, trash_root, trash_ttl_days)?; + let recovered = recover(journal_path)?; if let Ok(locator) = locator_path(repo) { let _ = std::fs::remove_file(locator); } @@ -656,10 +644,8 @@ fn reconcile_exchange_failure( /// Moves the replaced tree out of the way and returns where it landed. /// -/// The trash lives in the store, which can be on another volume than the -/// repo; a cross-device rename fails rather than copying, so a sibling of -/// the repo is the fallback. Either way the tree is off the repo path. -fn park_replaced_tree(tmp: &Path, trash_root: &Path, parent: &Path, name: &str) -> Result { +/// The destination is a sibling so the rename stays on the repository volume. +fn park_replaced_tree(tmp: &Path, parent: &Path, name: &str) -> Result { let stamp = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_or(0, |duration| duration.as_nanos()); @@ -668,10 +654,6 @@ fn park_replaced_tree(tmp: &Path, trash_root: &Path, parent: &Path, name: &str) std::process::id(), PARK_SEQUENCE.fetch_add(1, Ordering::Relaxed) ); - let trashed = trash_root.join(format!("{name}-{unique}")); - if durable_rename(tmp, &trashed, false).is_ok() { - return Ok(trashed); - } let sibling = parent.join(format!(".{name}.{}-trash-{unique}", crate::product::NAME)); durable_rename(tmp, &sibling, false).map_err(|error| { EngineError::Restore(format!( @@ -684,11 +666,7 @@ fn park_replaced_tree(tmp: &Path, trash_root: &Path, parent: &Path, name: &str) /// Startup crash recovery. Reads the journal (if any) and finishes or unwinds /// the interrupted rewind so the repo is whole before the pipeline baselines. -pub fn recover( - journal_path: &Path, - trash_root: &Path, - trash_ttl_days: u32, -) -> Result> { +pub fn recover(journal_path: &Path) -> Result> { let text = match std::fs::read_to_string(journal_path) { Ok(text) => text, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), @@ -773,12 +751,12 @@ pub fn recover( .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))? .to_string_lossy(); if path_exists(&journal.tmp)? { - old_tree = Some(park_replaced_tree(&journal.tmp, trash_root, parent, &name)?); + old_tree = Some(park_replaced_tree(&journal.tmp, parent, &name)?); } #[cfg(windows)] if let Some(scratch) = scratch { if path_exists(&scratch)? { - old_tree = Some(park_replaced_tree(&scratch, trash_root, parent, &name)?); + old_tree = Some(park_replaced_tree(&scratch, parent, &name)?); } } } @@ -786,7 +764,6 @@ pub fn recover( std::fs::remove_file(journal_path)?; #[cfg(unix)] sync_parent(journal_path)?; - prune_trash(trash_root, trash_ttl_days); Ok(Some(RecoveredSwap { published, old_tree, @@ -841,20 +818,13 @@ fn write_locator(path: &Path, locator: &RecoveryLocator) -> Result<()> { Ok(()) } -fn publish_locator( - repo: &Path, - journal: &Path, - trash: &Path, - trash_ttl_days: u32, -) -> Result { +fn publish_locator(repo: &Path, journal: &Path) -> Result { let path = locator_path(repo)?; write_locator( &path, &RecoveryLocator { repo_root: repo.to_path_buf(), journal: journal.to_path_buf(), - trash: trash.to_path_buf(), - trash_ttl_days, }, )?; Ok(path) @@ -919,12 +889,26 @@ fn sync_parent(path: &Path) -> Result<()> { Ok(()) } -fn prune_trash(trash_root: &Path, ttl_days: u32) { - let Ok(entries) = std::fs::read_dir(trash_root) else { +fn prune_sibling_trash(repo: &Path, ttl_days: u32) { + let Some(parent) = repo.parent() else { + return; + }; + let Some(name) = repo.file_name() else { + return; + }; + let prefix = format!( + ".{}.{}-trash-", + name.to_string_lossy(), + crate::product::NAME + ); + let Ok(entries) = std::fs::read_dir(parent) else { return; }; let ttl = std::time::Duration::from_secs(u64::from(ttl_days) * 24 * 3600); for entry in entries.flatten() { + if !entry.file_name().to_string_lossy().starts_with(&prefix) { + continue; + } let Ok(metadata) = entry.metadata() else { continue; }; @@ -1067,14 +1051,12 @@ mod tests { #[test] fn parking_replaced_trees_never_reuses_a_name() -> Result<()> { let work = tempfile::tempdir()?; - let trash = work.path().join("trash"); - std::fs::create_dir(&trash)?; let mut parked = Vec::new(); for index in 0..3 { let tmp = work.path().join(format!("tmp-{index}")); std::fs::create_dir(&tmp)?; std::fs::write(tmp.join("old.txt"), index.to_string())?; - let destination = park_replaced_tree(&tmp, &trash, work.path(), "repo")?; + let destination = park_replaced_tree(&tmp, work.path(), "repo")?; assert_eq!( std::fs::read_to_string(destination.join("old.txt"))?, index.to_string() @@ -1119,17 +1101,21 @@ mod tests { } fn recover(journal_path: &Path) -> Result> { - let trash = journal_path.with_file_name("trash"); - std::fs::create_dir_all(&trash)?; - super::recover(journal_path, &trash, 30) + super::recover(journal_path) } - fn parked_tree_contains(journal_path: &Path, file: &str, expected: &[u8]) -> bool { - let trash = journal_path.with_file_name("trash"); - std::fs::read_dir(trash) + fn parked_tree_contains(repo: &Path, file: &str, expected: &[u8]) -> bool { + repo.parent() + .and_then(|parent| std::fs::read_dir(parent).ok()) .into_iter() .flatten() .filter_map(std::result::Result::ok) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .contains(".acyclic-trash-") + }) .any(|entry| std::fs::read(entry.path().join(file)).ok().as_deref() == Some(expected)) } @@ -1296,7 +1282,7 @@ mod tests { b"LIVE" ); assert!(!tmp.exists()); - assert!(parked_tree_contains(&journal_path, "file.txt", b"old")); + assert!(parked_tree_contains(&repo, "file.txt", b"old")); } #[test] @@ -1397,8 +1383,6 @@ mod tests { let store = work.path().join("custom-store"); std::fs::create_dir(&store).expect("custom store"); let journal_path = store.join("rewind-journal.json"); - let trash = store.join("trash"); - std::fs::create_dir(&trash).expect("trash"); write(&journal_path, &journal(&repo, &staged, Phase::Swapping)); let locator = locator_path(&repo).expect("locator path"); write_locator( @@ -1406,8 +1390,6 @@ mod tests { &RecoveryLocator { repo_root: repo.clone(), journal: journal_path.clone(), - trash, - trash_ttl_days: 30, }, ) .expect("locator"); @@ -1452,7 +1434,7 @@ mod tests { std::fs::read(repo.join(".env")).expect("carried data survived"), b"carried live data" ); - assert!(parked_tree_contains(&journal_path, "file.txt", b"new tree")); + assert!(parked_tree_contains(&repo, "file.txt", b"new tree")); } /// After rename #2 the new tree is published, and the old tree in @@ -1472,14 +1454,10 @@ mod tests { write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); let generation = GenerationId::new(acyclic_fs::Digest::ZERO); - let trash = journal_path.with_file_name("trash"); - std::fs::create_dir(&trash).expect("trash"); let outcome = reconcile_exchange_failure( generation, &repo, &journal_path, - &trash, - 30, EngineError::Restore("injected third rename failure".into()), ) .expect("published rewind is a success"); @@ -1495,6 +1473,6 @@ mod tests { ); assert!(!scratch.exists(), "scratch must not outlive recovery"); assert!(!journal_path.exists()); - assert!(parked_tree_contains(&journal_path, "file.txt", b"old tree")); + assert!(parked_tree_contains(&repo, "file.txt", b"old tree")); } } diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 4e05b29..b4caa58 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -166,12 +166,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { // Finish or unwind any rewind that a crash interrupted BEFORE the store // opens and the pipeline baselines; sweep fork dirs a dead daemon left // mounted (fork sessions do not survive the daemon). - rewind::recover( - &paths.rewind_journal(), - &paths.trash(), - config.trash_ttl_days, - ) - .map_err(|error| error.to_string())?; + rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?; lap("rewind journal recovery"); fork::sweep_stale_forks(&repo_root); lap("sweep stale forks"); diff --git a/tests/acceptance/cursor-e2e.sh b/tests/acceptance/cursor-e2e.sh index 274ed55..ba31cb8 100755 --- a/tests/acceptance/cursor-e2e.sh +++ b/tests/acceptance/cursor-e2e.sh @@ -12,8 +12,8 @@ # # Cursor's own session id (reported here as `session_id` in --output-format # json) is the same identifier its beforeShellExecution/afterFileEdit hooks -# send as `conversation_id` — that's the fallback `hook::Payload::session()` -# resolves to `session_id` for attribution (Cursor's own `sessionStart` +# send as `conversation_id`; `hook::Payload::session()` maps that field to +# the timeline session (Cursor's own `sessionStart` # event does send `session_id` directly). If Cursor ever splits these, the # timeline/diff assertions below will fail loudly rather than silently # mis-attribute. diff --git a/tests/acceptance/windows-smoke.sh b/tests/acceptance/windows-smoke.sh index ce6c725..b024c89 100644 --- a/tests/acceptance/windows-smoke.sh +++ b/tests/acceptance/windows-smoke.sh @@ -10,7 +10,7 @@ # (a daemon that inherits it never lets the caller see EOF) # - capture and diff under UTF-16LE names, non-ASCII included # - rewind, which renames the repo root and so trips every open handle -# - ProjFS-accelerated forks when available, with copy fallback otherwise +# - required ProjFS-accelerated forks # # Runs under Git Bash on a GitHub windows runner. ACYCLIC_BIN overrides the # binary (default: target/release/acyclic.exe). From 17e8b45e674de3d8b13186a6b26931bc7486bb7b Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 05:26:55 +0100 Subject: [PATCH 032/106] Make fork teardown transactional --- crates/acyclic/src/server.rs | 62 ++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index b4caa58..2e6e336 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -688,10 +688,7 @@ impl Server { .map(|(id, _)| id.clone()) .collect(); for id in scratch_ids { - self.forks.lock().await.remove(&id); - if let Err(error) = self.detach_route(&id).await { - eprintln!("{NAME} daemon: drop scratch fork {id}: {error}"); - } + self.remove_fork(&id).await?; } // The session that just ended is the one the NEXT session's // brief will describe, and nothing is asking for it yet: @@ -783,10 +780,7 @@ impl Server { Ok(proto::Reply::Forks(entries)) } proto::Op::ForkDrop { id } => { - let mut forks = self.forks.lock().await; - forks.remove(&id).ok_or(format!("no fork {id}"))?; - drop(forks); - self.detach_route(&id).await?; + self.remove_fork(&id).await?; Ok(proto::Reply::Unit) } proto::Op::Promote { id } => { @@ -848,10 +842,11 @@ impl Server { } Ok(landed) => landed, }; - // Landed: the fork is consumed. Drop its mount route. - if let Err(error) = self.detach_route(&id).await { - eprintln!("{NAME} daemon: discard fork {id} after promote: {error}"); - } + // Landed: the fork is consumed. A teardown failure must be + // visible because the route is still live and the fork must + // remain tracked until a later drop can finish it. + self.forks.lock().await.insert(id.clone(), fork); + self.remove_fork(&id).await?; match landed { Landed::Replayed { generation, @@ -1000,6 +995,17 @@ impl Server { Ok(()) } + /// Detaches a fork's live route before forgetting its state. If teardown + /// fails, the entry remains visible and a caller can retry the drop. + async fn remove_fork(&self, id: &str) -> Result<(), String> { + if !self.forks.lock().await.contains_key(id) { + return Err(format!("no fork {id}")); + } + self.detach_route(id).await?; + self.forks.lock().await.remove(id); + Ok(()) + } + /// Builds the mount source for a fork's shared checkout and routes it /// under the one native session (mounting it on the first route). async fn attach_route( @@ -1420,24 +1426,32 @@ impl Server { /// disappears) when the last route goes, freeing the FUSE-T pool slot. async fn detach_route(&self, id: &str) -> Result<(), String> { let mut mount = self.fork_mount.lock().await; + let route = route_name(id); + let last_route = mount.router.route_count() == 1; + if last_route { + if let Some(session) = mount.session.as_mut() { + tokio::task::block_in_place(|| session.stop()) + .map_err(|error| format!("unmount: {error:?}"))?; + } + mount.session.take(); + } // Dropping a route drops its CheckoutMountSource, which owns a tokio - // runtime — runtimes must never be dropped on an async worker. - tokio::task::block_in_place(|| mount.router.remove_route(&route_name(id))); + // runtime — runtimes must never be dropped on an async worker. For + // the last route, stop the fallible kernel session first so failure + // leaves both the route and fork state intact for an exact retry. + if !tokio::task::block_in_place(|| mount.router.remove_route(&route)) { + return Err(format!("fork {id} has no mount route")); + } // The kernel may hold a positive entry cache for the removed name // (FSKit caches until told otherwise): invalidate it eagerly. if let Some(session) = mount.session.as_ref() { - if let Err(error) = tokio::task::block_in_place(|| session.invalidate(&route_name(id))) - { - eprintln!("{NAME} daemon: invalidate {id}: {error:?}"); - } + tokio::task::block_in_place(|| session.invalidate(&route)) + .map_err(|error| format!("invalidate {id}: {error:?}"))?; } - if mount.router.is_empty() { - if let Some(mut session) = mount.session.take() { - tokio::task::block_in_place(|| session.stop()) - .map_err(|error| format!("unmount: {error:?}"))?; - } + if last_route { if let Some(root) = fork::forks_mount_root(&self.repo_root) { - let _ = std::fs::remove_dir_all(root); + std::fs::remove_dir_all(root) + .map_err(|error| format!("remove fork mount root: {error}"))?; } } Ok(()) From b804692e64718b079c4602e845c2905e9701c644 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 05:28:06 +0100 Subject: [PATCH 033/106] Fail closed on stale fork cleanup --- crates/acyclic/src/fork.rs | 73 ++++++++++++++++++++++-------------- crates/acyclic/src/server.rs | 2 +- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/crates/acyclic/src/fork.rs b/crates/acyclic/src/fork.rs index 5d11e56..58bd178 100644 --- a/crates/acyclic/src/fork.rs +++ b/crates/acyclic/src/fork.rs @@ -111,43 +111,52 @@ pub fn forks_mount_root(repo_root: &Path) -> Option { Some(forks_root(repo_root)?.join("mnt")) } -/// Best-effort cleanup of fork dirs left by a dead daemon: unmount anything -/// still attached, then remove the directories. Mount sessions do not -/// survive the daemon in v1. -pub fn sweep_stale_forks(repo_root: &Path) { +/// Removes the single routed mount and workspace directory a dead daemon +/// left behind. An absent root is the only successful no-op. +pub fn sweep_stale_forks(repo_root: &Path) -> Result<(), String> { let Some(root) = forks_root(repo_root) else { - return; - }; - let Ok(entries) = std::fs::read_dir(&root) else { - return; + return Err("repo root has no parent for fork workspaces".to_owned()); }; + if !root.try_exists().map_err(|error| error.to_string())? { + return Ok(()); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + let mount = root.join("mnt"); // FUSE-T's go-nfsv4 helpers outlive a killed daemon and wedge the // vendor's tiny shared NFS port pool for every future mount on the // host — reap any helper serving one of OUR workspaces first. #[cfg(target_os = "macos")] - { - let _ = std::process::Command::new("pkill") + if mount.try_exists().map_err(|error| error.to_string())? { + let status = std::process::Command::new("pkill") .arg("-f") - .arg(format!("go-nfsv4.*{}", root.display())) - .status(); - } - for entry in entries.flatten() { - let path = entry.path(); - #[cfg(target_os = "macos")] - let _ = std::process::Command::new("umount") + .arg(format!("go-nfsv4.*{}", mount.display())) + .status() + .map_err(|error| format!("stop stale NFS helper: {error}"))?; + if !status.success() && status.code() != Some(1) { + return Err(format!("stop stale NFS helper: {status}")); + } + let status = std::process::Command::new("umount") .arg("-f") - .arg(&path) - .status(); - #[cfg(target_os = "linux")] - { - let _ = std::process::Command::new("fusermount") - .arg("-u") - .arg(&path) - .status(); + .arg(&mount) + .status() + .map_err(|error| format!("unmount stale fork workspace: {error}"))?; + if !status.success() { + return Err(format!("unmount stale fork workspace: {status}")); + } + } + #[cfg(target_os = "linux")] + if mount.try_exists().map_err(|error| error.to_string())? { + let status = std::process::Command::new("fusermount") + .arg("-u") + .arg(&mount) + .status() + .map_err(|error| format!("unmount stale fork workspace: {error}"))?; + if !status.success() { + return Err(format!("unmount stale fork workspace: {status}")); } - let _ = std::fs::remove_dir_all(&path); } - let _ = std::fs::remove_dir(&root); + std::fs::remove_dir_all(&root) + .map_err(|error| format!("remove stale fork workspace {}: {error}", root.display())) } #[cfg(test)] @@ -177,7 +186,15 @@ mod tests { std::fs::create_dir_all(root.join("dead-fork")).expect("stale"); std::fs::write(root.join("dead-fork/leftover"), b"x").expect("file"); - sweep_stale_forks(&repo); + sweep_stale_forks(&repo).expect("sweep"); assert!(!root.exists()); } + + #[test] + fn sweeping_an_absent_root_is_a_no_op() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + std::fs::create_dir(&repo).expect("repo"); + sweep_stale_forks(&repo).expect("absent root"); + } } diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 2e6e336..a1ae191 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -168,7 +168,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { // mounted (fork sessions do not survive the daemon). rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?; lap("rewind journal recovery"); - fork::sweep_stale_forks(&repo_root); + fork::sweep_stale_forks(&repo_root)?; lap("sweep stale forks"); // Same reason as the fork sweep, and the same moment: a model run a // crashed daemon left behind is still running, and still billing. From b953b7b2f5cc07e3d54c867ec8e8d0c83e1267b1 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 06:53:34 +0100 Subject: [PATCH 034/106] Defer plugin baseline until repository use --- Cargo.lock | 39 +++++++++++++++++ crates/acyclic/src/pipeline.rs | 75 +++++++++++++++++++++++++++++--- crates/acyclic/tests/pipeline.rs | 40 ++++++++++++----- 3 files changed, 136 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59c8cc8..528b9bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,6 +48,7 @@ dependencies = [ "prost", "prost-types", "serde", + "serde_json", "thiserror", "tokio", "tonic", @@ -62,7 +63,9 @@ dependencies = [ name = "acyclic-native-runtime" version = "0.1.0" dependencies = [ + "block2", "bytes", + "dispatch2", "io-uring", "libc", "windows-sys 0.61.2", @@ -306,6 +309,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -527,6 +539,18 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1251,6 +1275,21 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + [[package]] name = "once_cell" version = "1.21.4" diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index d1589ae..703505f 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -45,6 +45,8 @@ fn fork_moved(base: GenerationId) -> PromoteOutcome { /// Pipeline state reported by `status`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum State { + /// Metadata and watcher are open; no repository descendants were scanned. + NeedsBaseline, Baselining, Ready, Rewinding, @@ -223,6 +225,18 @@ enum Request { }, } +impl Request { + const fn requires_ready(&self) -> bool { + !matches!( + self, + Self::Status { .. } + | Self::TurnStarted { .. } + | Self::RecordGeneration { .. } + | Self::Shutdown { .. } + ) + } +} + /// Cloneable handle used by the daemon to talk to the pipeline. #[derive(Clone)] pub struct PipelineHandle { @@ -742,8 +756,9 @@ async fn run(store: Store, index: Index, config: Config, mut receiver: mpsc::Rec clippy::match_same_arms, reason = "the arms look identical but each `reply` is a differently typed sender" )] -fn fail_request(request: Request, message: &str) { - let error = || EngineError::Store(message.to_owned()); +fn fail_request(request: Request, error: impl std::fmt::Display) { + let message = error.to_string(); + let error = || EngineError::Store(message.clone()); match request { Request::Checkpoint { reply, .. } => drop(reply.send(Err(error()))), Request::Commit { reply } => drop(reply.send(Err(error()))), @@ -798,7 +813,7 @@ impl Pipeline { }; let exclusions = Exclusions::parse(&config.exclude)?; - let mut pipeline = Self { + let pipeline = Self { store, index, config, @@ -806,7 +821,7 @@ impl Pipeline { options, exclusions, cancel, - state: State::Baselining, + state: State::NeedsBaseline, last_generation: GenerationId::new(acyclic_fs::Digest::ZERO), last_checkpoint_row: None, checkpoints_since_commit: 0, @@ -814,10 +829,16 @@ impl Pipeline { watcher_health: WatcherHealth::default(), auto_pending: None, }; - pipeline.baseline(CheckpointKind::Baseline).await?; Ok(pipeline) } + async fn ensure_ready(&mut self) -> Result<()> { + if self.state == State::Ready { + return Ok(()); + } + self.baseline(CheckpointKind::Baseline).await + } + /// Full baseline: capture the whole tree, checkpoint, finish the watcher /// rescan, publish. Used at startup and after RescanRequired/rewind. #[allow( @@ -952,6 +973,12 @@ impl Pipeline { )] async fn handle(&mut self, request: Request) -> bool { self.last_activity = Instant::now(); + if request.requires_ready() { + if let Err(error) = self.ensure_ready().await { + fail_request(request, error); + return false; + } + } match request { Request::Checkpoint { kind, @@ -1464,7 +1491,13 @@ impl Pipeline { /// meantime. Never records a row when nothing changed, so it cannot spam /// the timeline. async fn auto_checkpoint(&mut self) { - if self.config.auto_checkpoint_idle_ms == 0 || self.state != State::Ready { + if self.config.auto_checkpoint_idle_ms == 0 { + return; + } + if self.state == State::NeedsBaseline && self.ensure_ready().await.is_err() { + return; + } + if self.state != State::Ready { return; } // Like `idle_commit`, failures here are advisory: the pending state @@ -2126,3 +2159,33 @@ mod root_hint_tests { assert!(matches!(out, WatchBatch::RescanRequired { .. })); } } + +#[cfg(test)] +mod readiness_tests { + use super::*; + + #[test] + fn metadata_requests_do_not_force_a_repository_scan() { + let (status_reply, _) = oneshot::channel(); + assert!(!Request::Status { + reply: status_reply + } + .requires_ready()); + + let (turn_reply, _) = oneshot::channel(); + assert!(!Request::TurnStarted { + session_id: "session".to_owned(), + prompt: "prompt".to_owned(), + reply: turn_reply, + } + .requires_ready()); + + let (checkpoint_reply, _) = oneshot::channel(); + assert!(Request::Checkpoint { + kind: CheckpointKind::Manual, + attribution: Attribution::default(), + reply: checkpoint_reply, + } + .requires_ready()); + } +} diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index 12ab776..1cb7f94 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -188,10 +188,19 @@ fn idle_timer_auto_checkpoints_changes_no_host_asked_for() { let (handle, thread) = pipeline::spawn(store, index, config); runtime.block_on(async { - // A round trip first, so baseline capture is guaranteed done before - // the edit — otherwise the edit can race into the baseline itself - // and leave nothing pending for the idle timer to find. - let baseline_row = handle.status().await.expect("status").last_checkpoint; + // Wait for the enabled background observer to establish its baseline + // before editing; status itself remains scan free. + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + let baseline_row = loop { + if let Some(row) = handle.status().await.expect("status").last_checkpoint { + break Some(row); + } + assert!( + tokio::time::Instant::now() < deadline, + "baseline never completed" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + }; // No hook, no explicit checkpoint call — just an edit, like a host // with no lifecycle-hook API would produce. @@ -258,10 +267,12 @@ fn zero_auto_checkpoint_idle_ms_disables_the_idle_timer() { let (handle, thread) = pipeline::spawn(store, index, config); runtime.block_on(async { - // Baseline done first, so the edit is guaranteed to be pending - // rather than absorbed into the baseline (which would pass this - // test for the wrong reason). - handle.status().await.expect("status"); + // An explicit checkpoint is the readiness boundary when background + // auto capture is disabled; status deliberately remains scan free. + handle + .checkpoint(CheckpointKind::Manual, Attribution::default()) + .await + .expect("baseline checkpoint"); std::fs::write(repo.path().join("a.txt"), b"two\n").expect("edit"); tokio::time::sleep(Duration::from_millis(500)).await; handle.shutdown().await.expect("shutdown"); @@ -269,9 +280,10 @@ fn zero_auto_checkpoint_idle_ms_disables_the_idle_timer() { thread.join().expect("pipeline thread"); let index = read_only_index(&paths.index_db()); - // Only the baseline row from init: the idle timer never ran. + // The explicit readiness request may be a baseline or a noop immediately + // after it; the disabled idle timer must add no later row. let latest = index.latest().expect("query").expect("a row exists"); - assert_eq!(latest.kind, CheckpointKind::Baseline); + assert_eq!(latest.kind, CheckpointKind::Noop); } /// An idle tick that has drained an edit into the checkout but not yet @@ -385,8 +397,12 @@ fn enqueued_checkpoint_survives_immediate_shutdown() { let (handle, thread) = pipeline::spawn(store, index, fast_config()); runtime.block_on(async { - // Sync on Ready first: a write issued during the startup baseline is - // captured by it, and the enqueued checkpoint would be a noop. + // An explicit checkpoint is the readiness boundary; status remains + // metadata only and does not scan the repository. + handle + .checkpoint(CheckpointKind::Manual, Attribution::default()) + .await + .expect("baseline checkpoint"); let status = handle.status().await.expect("status"); assert_eq!(status.state, pipeline::State::Ready); std::fs::write(repo.path().join("file.txt"), b"after\n").expect("edit"); From 4889301b5fbab4547760d77c9782488298ab8885 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 07:04:10 +0100 Subject: [PATCH 035/106] Defer plugin cleanup until first use --- crates/acyclic/src/server.rs | 25 ++++++++++++++++--------- crates/acyclic/src/spec_runner.rs | 8 +++----- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index a1ae191..413561b 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -163,17 +163,10 @@ pub fn run(repo_root: &Path) -> Result<(), String> { let paths = StorePaths::for_repo(&repo_root, stores_root.as_deref()) .map_err(|error| error.to_string())?; - // Finish or unwind any rewind that a crash interrupted BEFORE the store - // opens and the pipeline baselines; sweep fork dirs a dead daemon left - // mounted (fork sessions do not survive the daemon). + // Finish or unwind any rewind that a crash interrupted before serving + // stateful operations. Fork cleanup is delayed until forks are requested. rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?; lap("rewind journal recovery"); - fork::sweep_stale_forks(&repo_root)?; - lap("sweep stale forks"); - // Same reason as the fork sweep, and the same moment: a model run a - // crashed daemon left behind is still running, and still billing. - crate::spec_runner::sweep_stale_runs(&paths.spec_runs()); - lap("sweep stale spec runs"); let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; // Socket + pidfile FIRST, before the store opens: a client can then @@ -220,6 +213,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { router: Arc::new(RoutedMountSource::new()), session: None, })), + fork_cleanup_pending: Arc::new(Mutex::new(true)), spec, live_sessions: Arc::new(Mutex::new(HashSet::new())), last_activity: Arc::new(Mutex::new(Instant::now())), @@ -249,6 +243,7 @@ struct Server { shutdown: Arc, forks: Arc>>, fork_mount: Arc>, + fork_cleanup_pending: Arc>, /// The speculation scheduler, when it is enabled. `None` makes every /// call site a no-op, so the default path costs nothing. spec: Option>, @@ -263,6 +258,17 @@ struct Server { } impl Server { + async fn ensure_fork_workspace_clean(&self) -> Result<(), String> { + let mut pending = self.fork_cleanup_pending.lock().await; + if !*pending { + return Ok(()); + } + let repo_root = self.repo_root.clone(); + tokio::task::block_in_place(|| fork::sweep_stale_forks(&repo_root))?; + *pending = false; + Ok(()) + } + async fn serve(&self, stream: ipc::ServerStream) { let (read, mut write) = tokio::io::split(stream); let mut lines = BufReader::new(read).lines(); @@ -739,6 +745,7 @@ impl Server { fork::mount_setup_hint() )); } + self.ensure_fork_workspace_clean().await?; let root = fork::forks_mount_root(&self.repo_root) .ok_or("repo root has no parent for fork workspaces")?; let mut created = Vec::new(); diff --git a/crates/acyclic/src/spec_runner.rs b/crates/acyclic/src/spec_runner.rs index 9887e4c..335f90b 100644 --- a/crates/acyclic/src/spec_runner.rs +++ b/crates/acyclic/src/spec_runner.rs @@ -351,7 +351,7 @@ impl Drop for OwnedJob { } } -/// Kills anything a dead daemon left running, before the store opens. +/// Kills anything a dead daemon left running when speculation is enabled. /// /// Matched on the recorded command name as well as the pid, so a reused pid /// belonging to something else is never signalled — the check costs one @@ -362,7 +362,7 @@ impl Drop for OwnedJob { reason = "kill(pid, 0) only tests for the process's existence; the \ killpg that follows is guarded by a command-name match" )] -pub fn sweep_stale_runs(spec_runs: &Path) { +fn sweep_stale_runs(spec_runs: &Path) { let pids = spec_runs.join("pids"); let Ok(entries) = std::fs::read_dir(&pids) else { return; @@ -401,9 +401,6 @@ pub fn sweep_stale_runs(spec_runs: &Path) { /// Windows needs no sweep: each run's job object carries /// `KILL_ON_JOB_CLOSE` and the daemon holds its only handle, so a daemon /// that dies — however it dies — takes the run's whole tree with it. -#[cfg(not(unix))] -pub fn sweep_stale_runs(_spec_runs: &Path) {} - /// Whether the live process really is the run we recorded, rather than /// whatever inherited its pid. #[cfg(unix)] @@ -593,6 +590,7 @@ mod tests { }); } + #[cfg(unix)] #[test] fn sweeping_an_absent_directory_is_a_no_op() { let dir = tempfile::tempdir().expect("tempdir"); From 5831cae06bab9e85b688562f668e5969a04ec324 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 07:17:31 +0100 Subject: [PATCH 036/106] Use SDK text merge semantics --- Cargo.lock | 2 +- crates/acyclic/Cargo.toml | 1 - crates/acyclic/src/merge.rs | 172 ++++++------------------------------ 3 files changed, 29 insertions(+), 146 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 528b9bf..77da099 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,7 +10,6 @@ dependencies = [ "blake3", "bytes", "clap", - "diffy", "hex", "libc", "rmcp", @@ -38,6 +37,7 @@ dependencies = [ "cap-primitives", "cap-std", "cc", + "diffy", "fs2", "fuser", "futures", diff --git a/crates/acyclic/Cargo.toml b/crates/acyclic/Cargo.toml index 3d4d5e9..db549fe 100644 --- a/crates/acyclic/Cargo.toml +++ b/crates/acyclic/Cargo.toml @@ -22,7 +22,6 @@ thiserror = "2" hex = "0.4" blake3 = "1" bytes = "1" -diffy = "0.4" tokio = { version = "1.48", features = ["rt-multi-thread", "macros", "net", "io-util", "sync", "time", "signal", "process"] } [target.'cfg(unix)'.dependencies] diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 99b3922..cd85dee 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -4,16 +4,13 @@ //! **H** ("theirs"), and the fork snapshot **F** ("ours"; the fork is //! `ours` because it merges *into* the mainline, graphcoder's convention). //! [`plan`] walks the union of changed paths, applies the entry-level -//! decision table, runs [`merge3`] on regular text files both sides +//! decision table, runs the SDK text driver on regular text files both sides //! changed, and returns everything the daemon needs to either land the //! merge or rebase the fork with conflict markers. Nothing here writes to //! the working tree. //! -//! `merge3`, its trailing-newline rule, and [`has_conflict_markers`] are -//! verbatim ports of graphcoder's `lib/compute/src/merge.rs` and -//! `local/src/lib/worktree/conflict/markers.ts`. +//! The SDK owns marker and newline semantics so every consumer sees the same result. -use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -33,119 +30,10 @@ const PAGE_ENTRIES: u32 = 1_024; const BINARY_PROBE_BYTES: usize = 8 * 1024; // --------------------------------------------------------------------------- -// merge3: graphcoder's three-way text merge, ported verbatim +// Shared SDK text merge // --------------------------------------------------------------------------- -/// Result of a 3-way merge operation. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Merge3Result { - /// True if merge completed without conflicts. - pub clean: bool, - /// Merged content. If clean=false, contains conflict markers. - pub content: String, -} - -/// Ensures a string ends with a newline so conflict markers always occupy -/// complete lines. Borrows when nothing needs adding. -fn ensure_trailing_newline(s: &str) -> Cow<'_, str> { - if s.is_empty() || s.ends_with('\n') { - Cow::Borrowed(s) - } else { - Cow::Owned(format!("{s}\n")) - } -} - -/// Whether the clean result keeps a trailing newline: if ours and theirs -/// agree, that; else if ours agrees with base, theirs decides; else ours. -fn merged_has_trailing_newline(base: &str, ours: &str, theirs: &str) -> bool { - let base_has_newline = base.ends_with('\n'); - let ours_has_newline = ours.ends_with('\n'); - let theirs_has_newline = theirs.ends_with('\n'); - if ours_has_newline == theirs_has_newline { - ours_has_newline - } else if ours_has_newline == base_has_newline { - theirs_has_newline - } else { - ours_has_newline - } -} - -/// Performs a 3-way merge (diffy, diff3 conflict style, marker length 7). -/// -/// Conflict marker format: -/// ```text -/// <<<<<<< {ours_name} -/// ... ours content ... -/// ||||||| original -/// ... base content ... -/// ======= -/// ... theirs content ... -/// >>>>>>> {theirs_name} -/// ``` -pub fn merge3( - base: &str, - ours: &str, - theirs: &str, - ours_name: &str, - theirs_name: &str, -) -> Merge3Result { - use diffy::{ConflictStyle, MergeOptions}; - - let mut options = MergeOptions::new(); - options - .set_conflict_style(ConflictStyle::Diff3) - .set_conflict_marker_length(7); - - let base_norm = ensure_trailing_newline(base); - let ours_norm = ensure_trailing_newline(ours); - let theirs_norm = ensure_trailing_newline(theirs); - - match options.merge(&base_norm, &ours_norm, &theirs_norm) { - Ok(mut merged) => { - if !merged_has_trailing_newline(base, ours, theirs) { - merged.pop(); - } - Merge3Result { - clean: true, - content: merged, - } - } - Err(merged_with_conflicts) => { - let content = merged_with_conflicts - .replace("<<<<<<< ours", &format!("<<<<<<< {ours_name}")) - .replace(">>>>>>> theirs", &format!(">>>>>>> {theirs_name}")); - Merge3Result { - clean: false, - content, - } - } - } -} - -/// Whether `content` still holds a generated conflict block: a `<<<<<<< ` -/// marker at the start of a line AND a `=======` line. Recognizes the -/// `(modified)` / `(deleted)` label variants by construction. -pub fn has_conflict_markers(content: &str) -> bool { - let opens = content - .split_inclusive('\n') - .any(|line| line.starts_with("<<<<<<< ") || line.starts_with("<<<<<<<\t")); - if !opens { - return false; - } - content - .lines() - .any(|line| line == "=======" || line == "=======\r") -} - -/// Number of conflict blocks in marker-bearing content. -pub fn conflict_hunks(content: &str) -> u32 { - content - .lines() - .filter(|line| line.starts_with("<<<<<<< ")) - .count() - .try_into() - .unwrap_or(u32::MAX) -} +pub use acyclic_fs::text_merge::{conflict_hunks, has_conflict_markers}; // --------------------------------------------------------------------------- // Per-path decision table @@ -274,7 +162,7 @@ pub fn merge_file( if ours == theirs { return Ok(ContentMerge::Merged(ours.as_bytes().to_vec())); } - let result = merge3( + let result = acyclic_fs::text_merge::merge_text( base_text.unwrap_or(""), ours, theirs, @@ -319,8 +207,16 @@ fn modify_delete( theirs_label: &str, kind: ConflictKind, ) -> ContentMerge { - let base = ensure_trailing_newline(base); - let kept = ensure_trailing_newline(kept); + let base = if base.is_empty() || base.ends_with('\n') { + std::borrow::Cow::Borrowed(base) + } else { + std::borrow::Cow::Owned(format!("{base}\n")) + }; + let kept = if kept.is_empty() || kept.ends_with('\n') { + std::borrow::Cow::Borrowed(kept) + } else { + std::borrow::Cow::Owned(format!("{kept}\n")) + }; let (ours_block, theirs_block) = match kind { ConflictKind::TheirsDeleted => (kept.as_ref(), ""), _ => ("", kept.as_ref()), @@ -1174,7 +1070,7 @@ mod tests { let base = "line 1\nline 2\nline 3\n"; let ours = "OURS line 1\nline 2\nline 3\n"; let theirs = "line 1\nline 2\nTHEIRS line 3\n"; - let result = merge3(base, ours, theirs, "child", "parent"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "child", "parent"); assert!(result.clean); assert_eq!(result.content, "OURS line 1\nline 2\nTHEIRS line 3\n"); } @@ -1184,7 +1080,8 @@ mod tests { let base = "line 1\nline 2\nline 3\n"; let ours = "OURS line 1\nline 2\nline 3\n"; let theirs = "THEIRS line 1\nline 2\nline 3\n"; - let result = merge3(base, ours, theirs, "child-agent", "parent-agent"); + let result = + acyclic_fs::text_merge::merge_text(base, ours, theirs, "child-agent", "parent-agent"); assert!(!result.clean); assert!(result.content.contains("<<<<<<< child-agent\n")); assert!(result.content.contains("||||||| original\n")); @@ -1197,14 +1094,15 @@ mod tests { let base = "line 1\nline 2\n"; let ours = "SAME line 1\nline 2\n"; let theirs = "SAME line 1\nline 2\n"; - let result = merge3(base, ours, theirs, "child", "parent"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "child", "parent"); assert!(result.clean); assert_eq!(result.content, "SAME line 1\nline 2\n"); } #[test] fn conflict_markers_have_proper_newlines() { - let result = merge3("content", "ours", "theirs", "child", "parent"); + let result = + acyclic_fs::text_merge::merge_text("content", "ours", "theirs", "child", "parent"); assert!(!result.clean); assert!(result.content.contains("\n|||||||")); assert!(result.content.contains("\n=======\n")); @@ -1213,40 +1111,26 @@ mod tests { #[test] fn empty_base_ours_added_theirs_empty() { - let result = merge3("", "added", "", "child", "parent"); + let result = acyclic_fs::text_merge::merge_text("", "added", "", "child", "parent"); assert!(result.clean); assert_eq!(result.content, "added"); } #[test] fn empty_base_both_add_different() { - let result = merge3("", "ours\n", "theirs\n", "child", "parent"); + let result = + acyclic_fs::text_merge::merge_text("", "ours\n", "theirs\n", "child", "parent"); assert!(!result.clean); assert!(result.content.contains("<<<<<<< child")); assert!(result.content.contains(">>>>>>> parent")); } - // --- trailing newline rule -------------------------------------------- - - #[test] - fn trailing_newline_rule() { - // ours and theirs agree: keep theirs' (== ours') state. - assert!(merged_has_trailing_newline("a", "a\n", "b\n")); - assert!(!merged_has_trailing_newline("a\n", "a", "b")); - // ours == base, theirs decides. - assert!(!merged_has_trailing_newline("a\n", "a\n", "b")); - assert!(merged_has_trailing_newline("a", "a", "b\n")); - // ours differs from base and theirs: ours decides. - assert!(!merged_has_trailing_newline("a\n", "x", "b\n")); - assert!(merged_has_trailing_newline("a", "x\n", "b")); - } - #[test] fn fork_appends_without_trailing_newline_and_head_edits_top() { let base = "one\ntwo\nthree\n"; let ours = "one\ntwo\nthree\nfour"; let theirs = "ONE\ntwo\nthree\n"; - let result = merge3(base, ours, theirs, "fork", "mainline"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "fork", "mainline"); assert!(result.clean); assert_eq!(result.content, "ONE\ntwo\nthree\nfour"); } @@ -1256,7 +1140,7 @@ mod tests { let base = "a\r\nb\r\nc\r\n"; let ours = "A\r\nb\r\nc\r\n"; let theirs = "a\r\nb\r\nC\r\n"; - let result = merge3(base, ours, theirs, "fork", "mainline"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "fork", "mainline"); assert!(result.clean); assert_eq!(result.content, "A\r\nb\r\nC\r\n"); } @@ -1266,7 +1150,7 @@ mod tests { let base = "1\n2\n3\n4\n"; let ours = "1x\n2\n3\n4\n"; let theirs = "1\n2y\n3\n4\n"; - let result = merge3(base, ours, theirs, "fork", "mainline"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "fork", "mainline"); // Adjacent edits are a conflict for diff3 (git behaves the same); // whichever way diffy decides, the result must be consistent with clean. if result.clean { @@ -1281,7 +1165,7 @@ mod tests { let base = "a\nb\nc\nd\n"; let ours = "a\nd\n"; let theirs = "a\nB\nc\nd\n"; - let result = merge3(base, ours, theirs, "fork", "mainline"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "fork", "mainline"); assert!(!result.clean); assert_eq!(conflict_hunks(&result.content), 1); } From de7032a73070efe164df7d27575da406ca9b60ee Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 07:28:50 +0100 Subject: [PATCH 037/106] Use SDK bounded byte merge --- crates/acyclic/src/merge.rs | 145 +++++------------------------------- 1 file changed, 20 insertions(+), 125 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index cd85dee..522f07c 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -15,6 +15,12 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use acyclic_fs::kernel::{FileKind, FileMetadata, MetadataField, NamespacePath}; +pub use acyclic_fs::text_merge::{ + conflict_hunks, has_conflict_markers, ByteConflictKind as ConflictKind, +}; +use acyclic_fs::text_merge::{ + merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, +}; use acyclic_fs::{ByteRange, CancellationToken, GenerationId, WorkCounters}; use bytes::Bytes; @@ -27,14 +33,6 @@ use crate::{EngineError, Result}; pub const DEFAULT_MAX_FILE_BYTES: u64 = 4 * 1024 * 1024; const TRANSFER_BYTES: u64 = 8 * 1024 * 1024; const PAGE_ENTRIES: u32 = 1_024; -const BINARY_PROBE_BYTES: usize = 8 * 1024; - -// --------------------------------------------------------------------------- -// Shared SDK text merge -// --------------------------------------------------------------------------- - -pub use acyclic_fs::text_merge::{conflict_hunks, has_conflict_markers}; - // --------------------------------------------------------------------------- // Per-path decision table // --------------------------------------------------------------------------- @@ -102,39 +100,6 @@ impl std::fmt::Display for Reason { } } -/// What kind of conflict a marker-bearing file carries. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] -pub enum ConflictKind { - /// Overlapping hunks; `hunks` counts the blocks. - Hunks, - /// The fork modified a file the mainline deleted. - TheirsDeleted, - /// The mainline modified a file the fork deleted. - OursDeleted, -} - -/// Outcome of merging one regular file that both sides changed. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ContentMerge { - Merged(Vec), - Conflicted { - bytes: Vec, - hunks: u32, - kind: ConflictKind, - }, -} - -/// The text gate: UTF-8, no NUL in the probe window, under the size cap. -fn text_gate<'a>(bytes: &'a [u8], limits: &MergeLimits) -> std::result::Result<&'a str, Reason> { - if bytes.len() as u64 > limits.max_file_bytes { - return Err(Reason::TooLarge); - } - if bytes.iter().take(BINARY_PROBE_BYTES).any(|byte| *byte == 0) { - return Err(Reason::Binary); - } - std::str::from_utf8(bytes).map_err(|_| Reason::Binary) -} - /// Merges one regular file both sides changed. `None` on a side means that /// side deleted the file; `base` is `None` when both sides added it. pub fn merge_file( @@ -145,90 +110,20 @@ pub fn merge_file( theirs_name: &str, limits: &MergeLimits, ) -> std::result::Result { - fn gate<'a>( - side: Option<&'a [u8]>, - limits: &MergeLimits, - ) -> std::result::Result, Reason> { - match side { - None => Ok(None), - Some(bytes) => text_gate(bytes, limits).map(Some), - } - } - let base_text = gate(base, limits)?; - let ours_text = gate(ours, limits)?; - let theirs_text = gate(theirs, limits)?; - match (ours_text, theirs_text) { - (Some(ours), Some(theirs)) => { - if ours == theirs { - return Ok(ContentMerge::Merged(ours.as_bytes().to_vec())); - } - let result = acyclic_fs::text_merge::merge_text( - base_text.unwrap_or(""), - ours, - theirs, - ours_name, - theirs_name, - ); - if result.clean { - Ok(ContentMerge::Merged(result.content.into_bytes())) - } else { - let hunks = conflict_hunks(&result.content); - Ok(ContentMerge::Conflicted { - bytes: result.content.into_bytes(), - hunks, - kind: ConflictKind::Hunks, - }) - } - } - (Some(ours), None) => Ok(modify_delete( - base_text.unwrap_or(""), - ours, - &format!("{ours_name} (modified)"), - &format!("{theirs_name} (deleted)"), - ConflictKind::TheirsDeleted, - )), - (None, Some(theirs)) => Ok(modify_delete( - base_text.unwrap_or(""), - theirs, - &format!("{ours_name} (deleted)"), - &format!("{theirs_name} (modified)"), - ConflictKind::OursDeleted, - )), - (None, None) => Ok(ContentMerge::Merged(Vec::new())), - } -} - -/// A modify/delete conflict is always a conflict: one block holding the -/// surviving content against nothing, with the base in the middle. -fn modify_delete( - base: &str, - kept: &str, - ours_label: &str, - theirs_label: &str, - kind: ConflictKind, -) -> ContentMerge { - let base = if base.is_empty() || base.ends_with('\n') { - std::borrow::Cow::Borrowed(base) - } else { - std::borrow::Cow::Owned(format!("{base}\n")) - }; - let kept = if kept.is_empty() || kept.ends_with('\n') { - std::borrow::Cow::Borrowed(kept) - } else { - std::borrow::Cow::Owned(format!("{kept}\n")) - }; - let (ours_block, theirs_block) = match kind { - ConflictKind::TheirsDeleted => (kept.as_ref(), ""), - _ => ("", kept.as_ref()), - }; - let content = format!( - "<<<<<<< {ours_label}\n{ours_block}||||||| original\n{base}=======\n{theirs_block}>>>>>>> {theirs_label}\n" - ); - ContentMerge::Conflicted { - bytes: content.into_bytes(), - hunks: 1, - kind, - } + merge_bytes( + base, + ours, + theirs, + ours_name, + theirs_name, + ByteMergeLimits { + max_bytes: limits.max_file_bytes, + }, + ) + .map_err(|error| match error { + ByteMergeError::Binary => Reason::Binary, + ByteMergeError::TooLarge => Reason::TooLarge, + }) } // --------------------------------------------------------------------------- From 9f533673d11f5608c623933a11555b8a53a677b9 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 07:39:29 +0100 Subject: [PATCH 038/106] Use SDK whole-tree exchange for rewind --- crates/acyclic/src/rewind.rs | 152 ++++------------------------------- 1 file changed, 17 insertions(+), 135 deletions(-) diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index f27c756..ea0a0e7 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -8,7 +8,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use acyclic_fs::kernel::{LogicalName, NamespacePath}; use acyclic_fs::{ - materialize_checkout, materialize_checkout_host_path, MaterializeError, MaterializeOptions, + materialize_checkout, materialize_checkout_host_path, publish_native_exchange, + MaterializeError, MaterializeOptions, }; use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; use serde::{Deserialize, Serialize}; @@ -342,28 +343,8 @@ pub struct Journal { pub carried: Vec, } -/// Moves each `relative` path from `from` to `into`, replacing whatever the -/// materialized tree had there (the live copy wins). Stops at the first -/// failure; the caller unwinds with [`move_back`]. -fn carry(from: &Path, into: &Path, relative: &[PathBuf]) -> Result<()> { - for path in relative { - let source = from.join(path); - let destination = into.join(path); - if std::fs::symlink_metadata(&source).is_err() { - continue; - } - if let Some(parent) = destination.parent() { - std::fs::create_dir_all(parent)?; - } - let _ = remove_any(&destination); - std::fs::rename(&source, &destination).map_err(|error| { - EngineError::Restore(format!("carry excluded path {}: {error}", path.display())) - })?; - } - Ok(()) -} - -/// Inverse of [`carry`]: every listed path present in `from` goes back. +/// Returns every listed path present in `from` to `into` during legacy +/// journal recovery. /// A conflict or I/O failure keeps the journal and both trees for retry. fn move_back(from: &Path, into: &Path, relative: &[PathBuf]) -> Result<()> { for path in relative { @@ -552,55 +533,27 @@ pub async fn execute( EngineError::Restore(format!("materialize: {error:?}")) })?; - // 2. Carry the live excluded paths into the new tree. Journaled first, - // so a crash mid-carry can move them back. + // 2. Publish the complete sibling tree. The SDK owns durable exclusion + // carry and platform exchange recovery; this adapter owns materialization + // and the product's retained-tree naming policy. let carried: Vec = exclusions .host_paths() .into_iter() .filter(|relative| std::fs::symlink_metadata(repo.join(relative)).is_ok()) .collect(); - if !carried.is_empty() { - write_journal( - &journal_path, - &Journal { - target_generation: hex::encode(target.digest().as_bytes()), - repo_root: repo.clone(), - tmp: tmp.clone(), - phase: Phase::Carrying, - carried: carried.clone(), - }, - )?; - if let Err(error) = carry(repo, &tmp, &carried) { - // Undo what moved, then abandon the rewind with the repo whole. - move_back(&tmp, repo, &carried)?; - let _ = std::fs::remove_dir_all(&tmp); - let _ = std::fs::remove_file(&journal_path); - return Err(error); - } - } - - // 3. Atomic exchange: repo <-> tmp. After this the old tree is at `tmp`. - write_journal( - &journal_path, - &Journal { - target_generation: hex::encode(target.digest().as_bytes()), - repo_root: repo.clone(), - tmp: tmp.clone(), - phase: Phase::Swapping, - carried, - }, - )?; let locator = publish_locator(repo, &journal_path)?; - if let Err(error) = atomic_exchange(repo, &tmp) { - // The third Windows rename can fail after the new tree is already - // published. Reconcile immediately, before the daemon accepts another - // rewind that could reuse the scratch name. - return reconcile_exchange_failure(target, repo, &journal_path, error); + let exchange = publish_native_exchange(&journal_path, repo, &tmp, carried) + .map_err(|error| EngineError::Restore(format!("publish tree: {error}")))?; + if !exchange.published { + return Err(EngineError::Restore( + "native exchange recovered without publishing the target tree".into(), + )); } - // 4. Retain the old tree beside the repository. - let old_tree = park_replaced_tree(&tmp, parent, &name)?; - finish_published_rewind(&journal_path, &locator); + // 3. Retain the old tree beside the repository. + let displaced = exchange.displaced.unwrap_or(tmp); + let old_tree = park_replaced_tree(&displaced, parent, &name)?; + let _ = std::fs::remove_file(locator); prune_sibling_trash(repo, trash_ttl_days); Ok(RewindOutcome { @@ -610,38 +563,6 @@ pub async fn execute( }) } -fn finish_published_rewind(journal: &Path, locator: &Path) { - // The tree is already published. Cleanup failure leaves the journal and - // locator for startup retry, but must not report a failed rewind. - if std::fs::remove_file(journal).is_ok() { - let _ = std::fs::remove_file(locator); - } -} - -fn reconcile_exchange_failure( - target: GenerationId, - repo: &Path, - journal_path: &Path, - error: EngineError, -) -> Result { - let recovered = recover(journal_path)?; - if let Ok(locator) = locator_path(repo) { - let _ = std::fs::remove_file(locator); - } - if let Some(RecoveredSwap { - published: true, - old_tree: Some(old_tree), - }) = recovered - { - return Ok(RewindOutcome { - restored: target, - old_tree, - warning: "reload your editor: open files still point at the replaced tree", - }); - } - Err(error) -} - /// Moves the replaced tree out of the way and returns where it landed. /// /// The destination is a sibling so the rename stays on the repository volume. @@ -1436,43 +1357,4 @@ mod tests { ); assert!(parked_tree_contains(&repo, "file.txt", b"new tree")); } - - /// After rename #2 the new tree is published, and the old tree in - /// scratch must be parked rather than deleted. - #[cfg(windows)] - #[test] - fn recover_preserves_old_tree_after_second_windows_rename() { - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - std::fs::create_dir(&repo).expect("repo"); - std::fs::write(repo.join("file.txt"), b"new tree").expect("seed new"); - let tmp = work.path().join("repo.tmp"); - let scratch = swap_scratch(&repo).expect("scratch path"); - std::fs::create_dir(&scratch).expect("scratch"); - std::fs::write(scratch.join("file.txt"), b"old tree").expect("seed old"); - let journal_path = work.path().join("journal.json"); - write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); - - let generation = GenerationId::new(acyclic_fs::Digest::ZERO); - let outcome = reconcile_exchange_failure( - generation, - &repo, - &journal_path, - EngineError::Restore("injected third rename failure".into()), - ) - .expect("published rewind is a success"); - assert_eq!(outcome.restored, generation); - assert_eq!( - std::fs::read(outcome.old_tree.join("file.txt")).expect("old tree"), - b"old tree" - ); - - assert_eq!( - std::fs::read(repo.join("file.txt")).expect("repo whole"), - b"new tree" - ); - assert!(!scratch.exists(), "scratch must not outlive recovery"); - assert!(!journal_path.exists()); - assert!(parked_tree_contains(&repo, "file.txt", b"old tree")); - } } From 5b8f0a185ae4d422b52a6bfb77341e67e488a9f8 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 07:42:17 +0100 Subject: [PATCH 039/106] Use SDK native entry exchange --- crates/acyclic/src/rewind.rs | 188 +---------------------------------- 1 file changed, 4 insertions(+), 184 deletions(-) diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index ea0a0e7..b58521d 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -8,8 +8,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use acyclic_fs::kernel::{LogicalName, NamespacePath}; use acyclic_fs::{ - materialize_checkout, materialize_checkout_host_path, publish_native_exchange, - MaterializeError, MaterializeOptions, + exchange_native_entries, materialize_checkout, materialize_checkout_host_path, + publish_native_exchange, MaterializeError, MaterializeOptions, }; use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; use serde::{Deserialize, Serialize}; @@ -154,7 +154,8 @@ pub(crate) async fn materialize_path_into_checkout( PathReplace::Atomic => { // An exchange may have published the new node before a // durability error. Keep both staged and scratch trees. - atomic_exchange(&destination, &staged)?; + exchange_native_entries(&destination, &staged) + .map_err(|error| EngineError::Restore(format!("exchange path: {error}")))?; } PathReplace::LiveMount => replace_live_mount(&staged, &destination, parent)?, }, @@ -854,117 +855,6 @@ pub(crate) fn swap_scratch(a: &Path) -> Option { Some(parent.join(format!(".{name}.{}-swap", crate::product::NAME))) } -/// Exchanges two directories on the same filesystem. -/// -/// **Not atomic on Windows.** There is no `RENAME_EXCHANGE` equivalent: NTFS -/// cannot swap two names in one operation, so this is three renames through -/// a scratch name in `a`'s directory. Each rename is atomic; the sequence is -/// not, and a crash can be observed between any two of them. -/// -/// What still holds is the property rewind actually needs — the repo is -/// never a mixture of the two trees. Every intermediate state has the repo -/// path either absent or naming exactly one whole tree, and [`recover`] -/// resolves each of them from the journal: the `Swapping` arm finishes the -/// move when the repo path is missing, and clears the scratch either way. -/// A crash during a journal-less [`restore_path`] may leave the same scratch -/// behind; a later swap refuses to overwrite that potentially unique tree. -#[cfg(windows)] -fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { - let scratch = - swap_scratch(a).ok_or_else(|| EngineError::Restore("swap path has no parent".into()))?; - // A preexisting scratch may be the only copy of the prior tree after a - // failed exchange. The caller must recover it before starting a new swap. - if path_exists(&scratch)? { - return Err(EngineError::Restore(format!( - "swap scratch {} is occupied; recover the previous exchange first", - scratch.display() - ))); - } - - durable_rename(a, &scratch, false).map_err(|error| { - EngineError::Restore(format!("swap: move aside {}: {error}", a.display())) - })?; - if let Err(error) = durable_rename(b, a, false) { - // Nothing has been published yet; put `a` back and fail clean. - let _ = durable_rename(&scratch, a, false); - return Err(EngineError::Restore(format!( - "swap: move {} into place: {error}", - b.display() - ))); - } - durable_rename(&scratch, b, false).map_err(|error| { - // `a` already holds the new tree, so the exchange has effectively - // happened; only the old tree's parking spot is wrong. Leave the - // scratch for recovery rather than unwinding a published swap. - EngineError::Restore(format!("swap: park the replaced tree: {error}")) - }) -} - -/// Atomically exchanges two directories on the same filesystem. -#[cfg(target_os = "macos")] -#[allow( - unsafe_code, - reason = "renamex_np over two live NUL-terminated paths; RENAME_SWAP is atomic on APFS" -)] -fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { - use std::os::unix::ffi::OsStrExt; - let a_c = std::ffi::CString::new(a.as_os_str().as_bytes()) - .map_err(|_| EngineError::Restore("path contains NUL".into()))?; - let b_c = std::ffi::CString::new(b.as_os_str().as_bytes()) - .map_err(|_| EngineError::Restore("path contains NUL".into()))?; - // SAFETY: both are live NUL-terminated paths; RENAME_SWAP exchanges them - // atomically on APFS. - let result = unsafe { libc::renamex_np(a_c.as_ptr(), b_c.as_ptr(), libc::RENAME_SWAP) }; - if result == 0 { - sync_parent(a)?; - if a.parent() != b.parent() { - sync_parent(b)?; - } - Ok(()) - } else { - Err(EngineError::Restore(format!( - "renamex_np: {}", - std::io::Error::last_os_error() - ))) - } -} - -#[cfg(target_os = "linux")] -#[allow( - unsafe_code, - reason = "renameat2 over two live NUL-terminated paths; RENAME_EXCHANGE is atomic where supported" -)] -fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { - use std::os::unix::ffi::OsStrExt; - let a_c = std::ffi::CString::new(a.as_os_str().as_bytes()) - .map_err(|_| EngineError::Restore("path contains NUL".into()))?; - let b_c = std::ffi::CString::new(b.as_os_str().as_bytes()) - .map_err(|_| EngineError::Restore("path contains NUL".into()))?; - // SAFETY: both are live NUL-terminated paths; RENAME_EXCHANGE swaps them - // atomically on filesystems that support it. - let result = unsafe { - libc::syscall( - libc::SYS_renameat2, - libc::AT_FDCWD, - a_c.as_ptr(), - libc::AT_FDCWD, - b_c.as_ptr(), - libc::RENAME_EXCHANGE, - ) - }; - if result == 0 { - sync_parent(a)?; - if a.parent() != b.parent() { - sync_parent(b)?; - } - Ok(()) - } else { - Err(EngineError::Restore(format!( - "renameat2: {}", - std::io::Error::last_os_error() - ))) - } -} #[cfg(test)] mod tests { use super::*; @@ -1214,76 +1104,6 @@ mod tests { assert!(journal.carried.is_empty()); } - /// The contract every platform's exchange owes the caller, asserted - /// against whichever implementation this host compiled: after it, each - /// path names the other's tree. Windows reaches that through three - /// renames rather than one syscall, so it is the arm most worth pinning. - #[test] - fn exchange_swaps_two_directories() { - let work = tempfile::tempdir().expect("tempdir"); - let left = work.path().join("left"); - let right = work.path().join("right"); - std::fs::create_dir(&left).expect("left"); - std::fs::create_dir(&right).expect("right"); - std::fs::write(left.join("who.txt"), b"left").expect("seed left"); - std::fs::write(right.join("who.txt"), b"right").expect("seed right"); - - atomic_exchange(&left, &right).expect("exchange"); - - assert_eq!(std::fs::read(left.join("who.txt")).expect("left"), b"right"); - assert_eq!( - std::fs::read(right.join("who.txt")).expect("right"), - b"left" - ); - } - - /// A failed exchange must leave the tree it was given untouched rather - /// than half-moved. On Windows this exercises the unwind between the - /// first and second rename, which is the window where the repo path is - /// vacated and nothing has replaced it yet. - #[test] - fn a_failed_exchange_leaves_the_live_tree_whole() { - let work = tempfile::tempdir().expect("tempdir"); - let live = work.path().join("live"); - std::fs::create_dir(&live).expect("live"); - std::fs::write(live.join("keep.txt"), b"precious").expect("seed"); - let missing = work.path().join("never-materialized"); - - atomic_exchange(&live, &missing).expect_err("exchange must fail"); - - assert!(live.is_dir(), "the live tree must still be a directory"); - assert_eq!( - std::fs::read(live.join("keep.txt")).expect("content survives"), - b"precious" - ); - #[cfg(windows)] - assert!( - !swap_scratch(&live).expect("scratch path").exists(), - "a failed exchange must not leave its scratch behind" - ); - } - - #[cfg(windows)] - #[test] - fn exchange_refuses_to_overwrite_a_previous_scratch_tree() { - let work = tempfile::tempdir().expect("tempdir"); - let live = work.path().join("live"); - let staged = work.path().join("staged"); - let scratch = swap_scratch(&live).expect("scratch"); - for path in [&live, &staged, &scratch] { - std::fs::create_dir(path).expect("tree"); - } - std::fs::write(scratch.join("only-copy"), b"old data").expect("scratch data"); - - atomic_exchange(&live, &staged).expect_err("scratch must block new swap"); - assert!(live.exists()); - assert!(staged.exists()); - assert_eq!( - std::fs::read(scratch.join("only-copy")).expect("old data survived"), - b"old data" - ); - } - /// A crash after Windows rename #1 must roll back to the old tree and /// retain the staged new tree in trash for recovery or inspection. #[cfg(windows)] From 60d05019d943cd840a92ac9654e3ee690ecb5d47 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 07:44:31 +0100 Subject: [PATCH 040/106] Avoid full rescan after known tree publication --- crates/acyclic/src/pipeline.rs | 77 +++++++++++++++++++++++++++++++++- crates/acyclic/src/rewind.rs | 3 ++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 703505f..7c3aa70 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -2034,11 +2034,84 @@ impl Pipeline { &self.exclusions, ) .await; - self.reset_watch().await?; - self.baseline(CheckpointKind::Recovered).await?; + if outcome.is_ok() { + self.acknowledge_materialized_generation(generation).await?; + } else { + self.reset_watch().await?; + self.baseline(CheckpointKind::Recovered).await?; + } outcome } + /// Establishes a watcher boundary around an exact generation that this + /// process just materialized. The checkout already authenticates every + /// byte of the baseline, so only changes racing watcher admission need + /// capture. Any uncertainty falls back to the ordinary full baseline. + async fn acknowledge_materialized_generation( + &mut self, + generation: GenerationId, + ) -> Result<()> { + let started = Instant::now(); + self.store.checkout = self + .store + .volume + .checkout( + acyclic_fs::model::GenerationSelector::Exact(generation), + crate::store::writable_head(), + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await + .map_err(EngineError::fs("reopen materialized generation"))? + .value; + self.reset_watch().await?; + let batch = self + .watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? + .finish_rescan() + .map_err(EngineError::fs("finish materialization boundary"))?; + match batch { + WatchBatch::RescanRequired { reason, .. } => { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some(reason.to_string()); + self.reset_watch().await?; + self.baseline(CheckpointKind::Recovered).await + } + batch @ WatchBatch::Changes { .. } => { + let (batch, root) = strip_root_hints(batch); + if root != RootHint::None { + self.reset_watch().await?; + return self.baseline(CheckpointKind::Recovered).await; + } + if let WatchBatch::Changes { ref changes, .. } = batch { + if !changes.is_empty() { + capture_watch_batch( + &mut self.store.checkout, + batch, + &self.options, + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await + .map_err(EngineError::fs("capture materialization tail"))?; + self.scrub_exclusions().await?; + } + } + let acknowledged = self.checkpoint_engine().await?; + self.last_generation = acknowledged; + self.state = State::Ready; + self.watcher_health.last_recovery_ms = crate::trace::ms(started); + crate::trace!( + "pipeline", + "materialized generation acknowledged in {:.1}ms without full rescan", + crate::trace::ms(started) + ); + Ok(()) + } + } + } + /// Reopens the watcher and recomputes the capture root identity — needed /// whenever the repo directory inode may have changed (after a rewind). async fn reset_watch(&mut self) -> Result<()> { diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index b58521d..d7943c2 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -542,6 +542,9 @@ pub async fn execute( .into_iter() .filter(|relative| std::fs::symlink_metadata(repo.join(relative)).is_ok()) .collect(); + // The staging-only legacy phase has served its purpose. From this point + // the SDK owns the durable carry/exchange journal at the same path. + std::fs::remove_file(&journal_path)?; let locator = publish_locator(repo, &journal_path)?; let exchange = publish_native_exchange(&journal_path, repo, &tmp, carried) .map_err(|error| EngineError::Restore(format!("publish tree: {error}")))?; From 0384799a09a8f6529f1aa8021cc73220e989a1db Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 07:52:08 +0100 Subject: [PATCH 041/106] Preserve exact rewind watcher admission --- crates/acyclic/src/pipeline.rs | 77 +--------------------------------- 1 file changed, 2 insertions(+), 75 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 7c3aa70..703505f 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -2034,82 +2034,9 @@ impl Pipeline { &self.exclusions, ) .await; - if outcome.is_ok() { - self.acknowledge_materialized_generation(generation).await?; - } else { - self.reset_watch().await?; - self.baseline(CheckpointKind::Recovered).await?; - } - outcome - } - - /// Establishes a watcher boundary around an exact generation that this - /// process just materialized. The checkout already authenticates every - /// byte of the baseline, so only changes racing watcher admission need - /// capture. Any uncertainty falls back to the ordinary full baseline. - async fn acknowledge_materialized_generation( - &mut self, - generation: GenerationId, - ) -> Result<()> { - let started = Instant::now(); - self.store.checkout = self - .store - .volume - .checkout( - acyclic_fs::model::GenerationSelector::Exact(generation), - crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, - ) - .await - .map_err(EngineError::fs("reopen materialized generation"))? - .value; self.reset_watch().await?; - let batch = self - .watch - .as_mut() - .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? - .finish_rescan() - .map_err(EngineError::fs("finish materialization boundary"))?; - match batch { - WatchBatch::RescanRequired { reason, .. } => { - self.watcher_health.invalidations += 1; - self.watcher_health.last_reason = Some(reason.to_string()); - self.reset_watch().await?; - self.baseline(CheckpointKind::Recovered).await - } - batch @ WatchBatch::Changes { .. } => { - let (batch, root) = strip_root_hints(batch); - if root != RootHint::None { - self.reset_watch().await?; - return self.baseline(CheckpointKind::Recovered).await; - } - if let WatchBatch::Changes { ref changes, .. } = batch { - if !changes.is_empty() { - capture_watch_batch( - &mut self.store.checkout, - batch, - &self.options, - WorkCounters::UNBOUNDED, - &self.cancel, - ) - .await - .map_err(EngineError::fs("capture materialization tail"))?; - self.scrub_exclusions().await?; - } - } - let acknowledged = self.checkpoint_engine().await?; - self.last_generation = acknowledged; - self.state = State::Ready; - self.watcher_health.last_recovery_ms = crate::trace::ms(started); - crate::trace!( - "pipeline", - "materialized generation acknowledged in {:.1}ms without full rescan", - crate::trace::ms(started) - ); - Ok(()) - } - } + self.baseline(CheckpointKind::Recovered).await?; + outcome } /// Reopens the watcher and recomputes the capture root identity — needed From ca0840f86fe7c3a801d2012deb2ecd835745439c Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 08:13:43 +0100 Subject: [PATCH 042/106] Skip unchanged Windows startup scans --- crates/acyclic/src/pipeline.rs | 165 +++++++++++++++++++++++++++++++-- crates/acyclic/src/store.rs | 48 ++++++++++ 2 files changed, 207 insertions(+), 6 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 703505f..474b6b1 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -33,6 +33,50 @@ const MAXIMUM_EXTENT_SPANS: u32 = 65_536; const WATCH_QUEUE: u32 = 65_536; const POLL_CHANGES: u32 = 16_384; +#[cfg(target_os = "windows")] +const CONTINUITY_MAGIC: &[u8; 8] = b"ACCONT\0\x01"; +#[cfg(target_os = "windows")] +const CONTINUITY_BYTES: usize = 88; + +#[cfg(target_os = "windows")] +#[derive(Clone, Copy)] +struct ContinuityRecord { + generation: GenerationId, + row: i64, + usn: acyclic_fs::WindowsUsnCheckpoint, +} + +#[cfg(target_os = "windows")] +impl ContinuityRecord { + fn encode(self) -> [u8; CONTINUITY_BYTES] { + let mut bytes = [0_u8; CONTINUITY_BYTES]; + bytes[..8].copy_from_slice(CONTINUITY_MAGIC); + bytes[8..40].copy_from_slice(self.generation.digest().as_bytes()); + bytes[40..48].copy_from_slice(&self.row.to_le_bytes()); + bytes[48..].copy_from_slice(&self.usn.to_bytes()); + bytes + } + + fn decode(bytes: &[u8]) -> Option { + if bytes.len() != CONTINUITY_BYTES || !bytes.starts_with(CONTINUITY_MAGIC) { + return None; + } + let mut generation = [0_u8; 32]; + generation.copy_from_slice(bytes.get(8..40)?); + let mut row = [0_u8; 8]; + row.copy_from_slice(bytes.get(40..48)?); + let row = i64::from_le_bytes(row); + if row <= 0 { + return None; + } + Some(Self { + generation: GenerationId::new(acyclic_fs::Digest::from_bytes(generation)), + row, + usn: acyclic_fs::WindowsUsnCheckpoint::from_bytes(bytes.get(48..)?).ok()?, + }) + } +} + fn fork_moved(base: GenerationId) -> PromoteOutcome { PromoteOutcome::Conflict { message: format!( @@ -800,9 +844,15 @@ impl Pipeline { }, ) .map_err(EngineError::fs("open watcher"))?; - watch - .begin_rescan() - .map_err(EngineError::fs("begin rescan"))?; + #[cfg(target_os = "windows")] + let admitted = Self::admit_continuity(&store, &index, &mut watch)?; + #[cfg(not(target_os = "windows"))] + let admitted = false; + if !admitted { + watch + .begin_rescan() + .map_err(EngineError::fs("begin rescan"))?; + } let options = CaptureOptions { source_root: repo_root.clone(), @@ -813,6 +863,7 @@ impl Pipeline { }; let exclusions = Exclusions::parse(&config.exclude)?; + let latest = admitted.then(|| index.latest()).transpose()?.flatten(); let pipeline = Self { store, index, @@ -821,9 +872,17 @@ impl Pipeline { options, exclusions, cancel, - state: State::NeedsBaseline, - last_generation: GenerationId::new(acyclic_fs::Digest::ZERO), - last_checkpoint_row: None, + state: if admitted { + State::Ready + } else { + State::NeedsBaseline + }, + last_generation: latest + .as_ref() + .map_or(GenerationId::new(acyclic_fs::Digest::ZERO), |row| { + row.generation + }), + last_checkpoint_row: latest.map(|row| row.id), checkpoints_since_commit: 0, last_activity: Instant::now(), watcher_health: WatcherHealth::default(), @@ -832,6 +891,34 @@ impl Pipeline { Ok(pipeline) } + #[cfg(target_os = "windows")] + fn admit_continuity(store: &Store, index: &Index, watch: &mut NativeWatch) -> Result { + let bytes = match std::fs::read(store.paths.continuity()) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(_) => return Ok(false), + }; + let Some(record) = ContinuityRecord::decode(&bytes) else { + return Ok(false); + }; + let Some(row) = index.latest()? else { + return Ok(false); + }; + if !row.published + || row.id != record.row + || row.generation != record.generation + || store.checkout.generation_id() != record.generation + { + return Ok(false); + } + Ok(matches!( + watch + .accept_windows_usn_baseline(record.usn) + .map_err(EngineError::fs("admit Windows continuity"))?, + Ok(acyclic_fs::WindowsUsnContinuity::Unchanged) + )) + } + async fn ensure_ready(&mut self) -> Result<()> { if self.state == State::Ready { return Ok(()); @@ -855,6 +942,8 @@ impl Pipeline { // A newly created hard link can race the rescan tail. Restart with a // fresh watcher so the next full capture sees every alias together. for attempt in 0..3 { + #[cfg(target_os = "windows")] + let continuity = acyclic_fs::capture_windows_usn_checkpoint(&self.store.repo_root).ok(); // A baseline requires a clean checkout. Mid-session (watcher // invalidation, rewind) the overlay holds uncommitted captures: // publish them first. At startup this is a no-op. @@ -941,6 +1030,18 @@ impl Pipeline { self.last_checkpoint_row = Some(row); let phase = Instant::now(); self.commit_engine().await?; + #[cfg(target_os = "windows")] + if let Some(usn) = continuity { + crate::store::durable_replace( + &self.store.paths.continuity(), + &ContinuityRecord { + generation, + row, + usn, + } + .encode(), + )?; + } let postcommit_ms = crate::trace::ms(phase); self.state = State::Ready; if kind == CheckpointKind::Recovered { @@ -2160,6 +2261,58 @@ mod root_hint_tests { } } +#[cfg(all(test, target_os = "windows"))] +mod continuity_tests { + use super::*; + + fn record() -> ContinuityRecord { + let mut bytes = [0_u8; 40]; + bytes[..8].copy_from_slice(b"ACYUSN\0\x01"); + bytes[8..24].copy_from_slice(&[7; 16]); + bytes[24..32].copy_from_slice(&19_u64.to_le_bytes()); + bytes[32..40].copy_from_slice(&23_u64.to_le_bytes()); + let usn = acyclic_fs::WindowsUsnCheckpoint::from_bytes(&bytes).expect("canonical fixture"); + ContinuityRecord { + generation: GenerationId::new(acyclic_fs::Digest::from_bytes([3; 32])), + row: 7, + usn, + } + } + + #[test] + fn continuity_record_is_canonical_and_rejects_torn_state() { + let record = record(); + let bytes = record.encode(); + let decoded = ContinuityRecord::decode(&bytes).expect("canonical record"); + assert_eq!(decoded.generation, record.generation); + assert_eq!(decoded.row, record.row); + assert_eq!(decoded.usn, record.usn); + assert!(ContinuityRecord::decode(&bytes[..CONTINUITY_BYTES - 1]).is_none()); + let mut wrong_version = bytes; + wrong_version[0] ^= 0xff; + assert!(ContinuityRecord::decode(&wrong_version).is_none()); + let mut invalid_row = bytes; + invalid_row[40..48].copy_from_slice(&0_i64.to_le_bytes()); + assert!(ContinuityRecord::decode(&invalid_row).is_none()); + } + + #[test] + fn durable_replacement_keeps_only_the_complete_new_record() { + let directory = tempfile::tempdir().expect("store"); + let path = directory.path().join("continuity.bin"); + let first = record().encode(); + crate::store::durable_replace(&path, &first).expect("first replace"); + let mut second = record(); + second.row += 1; + crate::store::durable_replace(&path, &second.encode()).expect("second replace"); + let stored = std::fs::read(path).expect("read marker"); + assert_eq!( + ContinuityRecord::decode(&stored).expect("complete").row, + second.row + ); + } +} + #[cfg(test)] mod readiness_tests { use super::*; diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index ab6d6c8..9ed7cec 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -4,6 +4,8 @@ //! tree and fail-closes on sockets). The repo carries only `.acyclic/config.toml`. use std::path::{Path, PathBuf}; +#[cfg(target_os = "windows")] +use std::{fs::OpenOptions, io::Write}; use acyclic_fs::model::{ AccessMode, CheckoutMode, ConsistencyMode, GenerationSelector, Lifecycle, MutationMode, @@ -74,6 +76,8 @@ impl StorePaths { self.rewind_journal(), self.trash(), self.meta(), + #[cfg(target_os = "windows")] + self.continuity(), self.root.join("daemon.log"), ] { let resolved = canonicalize_planned(&path)?; @@ -115,6 +119,10 @@ impl StorePaths { pub fn meta(&self) -> PathBuf { self.root.join("meta.json") } + #[cfg(target_os = "windows")] + pub fn continuity(&self) -> PathBuf { + self.root.join("continuity.bin") + } /// Speculation cache. Deliberately NOT a table in `index_db`: the /// pipeline thread owns that connection and writes to it synchronously, /// so a second writer contending for `SQLite`'s write lock would block the @@ -394,6 +402,46 @@ fn atomic_write_json(path: &Path, value: &T) -> Result<()> { Ok(()) } +#[cfg(target_os = "windows")] +pub(crate) fn durable_replace(path: &Path, bytes: &[u8]) -> Result<()> { + let tmp = path.with_extension("bin.tmp"); + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::{ + MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, + }; + let from: Vec = tmp + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let to: Vec = path + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + // SAFETY: both paths are terminated UTF-16 buffers live for this call. + #[allow(unsafe_code)] + let replaced = unsafe { + MoveFileExW( + from.as_ptr(), + to.as_ptr(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, + ) + }; + if replaced == 0 { + return Err(std::io::Error::last_os_error().into()); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; From a9ab212bc948174ccbca7e489d9e07cef7ab8769 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 08:42:34 +0100 Subject: [PATCH 043/106] Centralize rewind head restoration in SDK --- crates/acyclic/src/pipeline.rs | 54 +++++++++++- crates/acyclic/src/rewind.rs | 154 +++++++++++++++++++++++++++------ crates/acyclic/src/server.rs | 9 +- crates/acyclic/src/store.rs | 78 ++++++++++++++--- 4 files changed, 253 insertions(+), 42 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 474b6b1..79b64ca 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -13,7 +13,7 @@ use acyclic_fs::{capture_baseline, capture_root_identity, capture_watch_batch, C use acyclic_fs::{ CancellationToken, CheckoutCommitOutcome, GenerationId, MountPublication, NativeWatch, NativeWatchOptions, OperationId, WatchBatch, WatchChange, WatchEpoch, WatchSequence, - WorkCounters, + WorkCounters, WorkspaceRestore, }; use tokio::sync::{mpsc, oneshot}; @@ -2128,13 +2128,61 @@ impl Pipeline { generation: GenerationId, ) -> Result { drop(self.watch.take()); - let outcome = rewind::execute( + let prepared = rewind::prepare( &self.store, generation, self.config.trash_ttl_days, &self.exclusions, ) - .await; + .await?; + let current = self + .store + .workspace + .head() + .await + .map_err(EngineError::fs("workspace head before rewind"))?; + let target = self + .store + .workspace + .generation(generation) + .await + .map_err(EngineError::fs("rewind generation"))?; + prepared.mark_restoring_head()?; + let key_bytes: [u8; 16] = generation.digest().as_bytes()[..16] + .try_into() + .map_err(|_| EngineError::Store("invalid generation digest".into()))?; + match self + .store + .workspace + .restore_generation( + &target, + current.id(), + acyclic_fs::IdempotencyKey::from_bytes(key_bytes), + ) + .await + .map_err(EngineError::fs("restore workspace generation"))? + { + WorkspaceRestore::Restored(_) + | WorkspaceRestore::AlreadyRestored(_) + | WorkspaceRestore::Current(_) => {} + WorkspaceRestore::Stale(_) + | WorkspaceRestore::Fenced + | WorkspaceRestore::IdempotencyConflict => { + return Err(EngineError::Store( + "workspace head changed during rewind".into(), + )); + } + } + let outcome = prepared.publish(); + self.store.checkout = self + .store + .workspace + .checkout( + acyclic_fs::model::GenerationSelector::Head, + crate::store::writable_head(), + ) + .await + .map_err(EngineError::fs("refresh rewind checkout"))?; self.reset_watch().await?; self.baseline(CheckpointKind::Recovered).await?; outcome diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index d7943c2..3b6eb5d 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -33,6 +33,17 @@ pub struct RewindOutcome { pub warning: &'static str, } +pub(crate) struct PreparedRewind<'a> { + store: &'a Store, + target: GenerationId, + tmp: PathBuf, + parent: PathBuf, + name: String, + journal_path: PathBuf, + carried: Vec, + trash_ttl_days: u32, +} + /// What a single-path restore did to the working tree. #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -372,6 +383,8 @@ fn move_back(from: &Path, into: &Path, relative: &[PathBuf]) -> Result<()> { pub enum Phase { /// Materializing into tmp; repo untouched. Recovery: delete tmp. Materializing, + /// The prepared tree is complete and the SDK head is being restored. + RestoringHead, /// Excluded paths moving from repo into tmp. Recovery: move back any /// that already moved, then delete tmp. Carrying, @@ -387,6 +400,52 @@ pub struct RecoveredSwap { pub published: bool, /// The displaced tree retained beside the repository. pub old_tree: Option, + /// Generation named by the durable journal. + pub target: GenerationId, +} + +pub async fn recover_workspace(store: &mut Store, recovered: RecoveredSwap) -> Result<()> { + store.recover_workspace_head(&recovered).await?; + if recovered.published { + return Ok(()); + } + let text = match std::fs::read_to_string(store.paths.rewind_journal()) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + let journal: Journal = serde_json::from_str(&text) + .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; + if journal.phase != Phase::RestoringHead { + return Ok(()); + } + let locator = publish_locator(&journal.repo_root, &store.paths.rewind_journal())?; + let exchange = publish_native_exchange( + &store.paths.rewind_journal(), + &journal.repo_root, + &journal.tmp, + journal.carried, + ) + .map_err(|error| EngineError::Restore(format!("recover publish tree: {error}")))?; + if !exchange.published { + return Err(EngineError::Restore( + "rewind recovery did not publish prepared tree".into(), + )); + } + if let Some(displaced) = exchange.displaced { + let parent = journal + .repo_root + .parent() + .ok_or_else(|| EngineError::Restore("rewind repo root has no parent".into()))?; + let name = journal + .repo_root + .file_name() + .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))? + .to_string_lossy(); + let _ = park_replaced_tree(&displaced, parent, &name)?; + } + let _ = std::fs::remove_file(locator); + Ok(()) } /// Stored beside the repository so startup can find the journal even while @@ -472,12 +531,12 @@ pub fn recover_before_repo_open(repo_root: &Path) -> Result { /// captures paused; the caller re-baselines afterwards. Excluded paths are /// carried from the live tree into the restored one: no checkpoint holds /// them, so the working copy is the only copy. -pub async fn execute( - store: &Store, +pub(crate) async fn prepare<'a>( + store: &'a Store, target: GenerationId, trash_ttl_days: u32, exclusions: &Exclusions, -) -> Result { +) -> Result> { let repo = &store.repo_root; let parent = repo .parent() @@ -534,37 +593,62 @@ pub async fn execute( EngineError::Restore(format!("materialize: {error:?}")) })?; - // 2. Publish the complete sibling tree. The SDK owns durable exclusion - // carry and platform exchange recovery; this adapter owns materialization - // and the product's retained-tree naming policy. let carried: Vec = exclusions .host_paths() .into_iter() .filter(|relative| std::fs::symlink_metadata(repo.join(relative)).is_ok()) .collect(); - // The staging-only legacy phase has served its purpose. From this point - // the SDK owns the durable carry/exchange journal at the same path. - std::fs::remove_file(&journal_path)?; - let locator = publish_locator(repo, &journal_path)?; - let exchange = publish_native_exchange(&journal_path, repo, &tmp, carried) - .map_err(|error| EngineError::Restore(format!("publish tree: {error}")))?; - if !exchange.published { - return Err(EngineError::Restore( - "native exchange recovered without publishing the target tree".into(), - )); + Ok(PreparedRewind { + store, + target, + tmp, + parent: parent.to_path_buf(), + name, + journal_path, + carried, + trash_ttl_days, + }) +} + +impl PreparedRewind<'_> { + pub(crate) fn mark_restoring_head(&self) -> Result<()> { + write_journal( + &self.journal_path, + &Journal { + target_generation: hex::encode(self.target.digest().as_bytes()), + repo_root: self.store.repo_root.clone(), + tmp: self.tmp.clone(), + phase: Phase::RestoringHead, + carried: self.carried.clone(), + }, + ) } - // 3. Retain the old tree beside the repository. - let displaced = exchange.displaced.unwrap_or(tmp); - let old_tree = park_replaced_tree(&displaced, parent, &name)?; - let _ = std::fs::remove_file(locator); - prune_sibling_trash(repo, trash_ttl_days); + pub(crate) fn publish(self) -> Result { + let repo = &self.store.repo_root; + // The staging-only legacy phase has served its purpose. From this point + // the SDK owns the durable carry/exchange journal at the same path. + std::fs::remove_file(&self.journal_path)?; + let locator = publish_locator(repo, &self.journal_path)?; + let exchange = publish_native_exchange(&self.journal_path, repo, &self.tmp, self.carried) + .map_err(|error| EngineError::Restore(format!("publish tree: {error}")))?; + if !exchange.published { + return Err(EngineError::Restore( + "native exchange recovered without publishing the target tree".into(), + )); + } - Ok(RewindOutcome { - restored: target, - old_tree, - warning: "reload your editor: open files still point at the replaced tree", - }) + let displaced = exchange.displaced.unwrap_or(self.tmp); + let old_tree = park_replaced_tree(&displaced, &self.parent, &self.name)?; + let _ = std::fs::remove_file(locator); + prune_sibling_trash(repo, self.trash_ttl_days); + + Ok(RewindOutcome { + restored: self.target, + old_tree, + warning: "reload your editor: open files still point at the replaced tree", + }) + } } /// Moves the replaced tree out of the way and returns where it landed. @@ -599,6 +683,7 @@ pub fn recover(journal_path: &Path) -> Result> { }; let journal: Journal = serde_json::from_str(&text) .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; + let target = decode_generation(&journal.target_generation)?; #[cfg(windows)] let mut published = false; #[cfg(not(windows))] @@ -609,6 +694,13 @@ pub fn recover(journal_path: &Path) -> Result> { // Repo untouched; the partial tmp tree is garbage. let _ = std::fs::remove_dir_all(&journal.tmp); } + Phase::RestoringHead => { + return Ok(Some(RecoveredSwap { + published, + old_tree, + target, + })) + } Phase::Carrying => { // Some excluded paths may already sit in tmp: bring them home, // then drop the unused new tree. @@ -692,9 +784,19 @@ pub fn recover(journal_path: &Path) -> Result> { Ok(Some(RecoveredSwap { published, old_tree, + target, })) } +fn decode_generation(encoded: &str) -> Result { + let bytes = hex::decode(encoded) + .map_err(|error| EngineError::Restore(format!("rewind generation: {error}")))?; + let digest: [u8; 32] = bytes + .try_into() + .map_err(|_| EngineError::Restore("rewind generation must be 32 bytes".into()))?; + Ok(GenerationId::new(acyclic_fs::Digest::from_bytes(digest))) +} + fn path_exists(path: &Path) -> Result { match std::fs::symlink_metadata(path) { Ok(_) => Ok(true), diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 413561b..5dbfd02 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -165,7 +165,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { // Finish or unwind any rewind that a crash interrupted before serving // stateful operations. Fork cleanup is delayed until forks are requested. - rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?; + let recovered = rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?; lap("rewind journal recovery"); let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; @@ -176,9 +176,14 @@ pub fn run(repo_root: &Path) -> Result<(), String> { // is removed; a live one refuses the second daemon via bind failure. let listener = bind_socket(&runtime, &paths)?; lap("socket bind + pidfile"); - let store = runtime + let mut store = runtime .block_on(Store::open(&repo_root, paths.clone())) .map_err(|error| error.to_string())?; + if let Some(recovered) = recovered { + runtime + .block_on(rewind::recover_workspace(&mut store, recovered)) + .map_err(|error| error.to_string())?; + } let repo_root = store.repo_root.clone(); lap("store open"); let index = Index::open(&paths.index_db()).map_err(|error| error.to_string())?; diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index 9ed7cec..c3150b3 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -14,6 +14,7 @@ use acyclic_fs::model::{ use acyclic_fs::{ CancellationToken, Checkout, LocalAuthorityBackend, LocalFs, LocalObjectBackend, LocalObjectsDurability, LocalOptions, LocalStreamDurability, VolumeId, WorkCounters, + WorkspaceRestore, }; use serde::{Deserialize, Serialize}; @@ -23,6 +24,8 @@ use crate::{EngineError, Result}; pub type LocalCheckout = Checkout; /// Concrete volume type for the local backend. pub type LocalVolume = acyclic_fs::LocalVolume; +/// Concrete workspace type for the local backend. +pub type LocalWorkspace = acyclic_fs::Workspace; /// Batch limits sized for large monorepos: the fs defaults (2,048) reject any /// baseline capture beyond ~2k paths. Immutable per volume — size generously. @@ -177,6 +180,7 @@ pub struct StoreMeta { /// An opened store: the fs engine, its volume, and a writable Head checkout. pub struct Store { pub fs: LocalFs, + pub workspace: LocalWorkspace, pub volume: LocalVolume, pub checkout: LocalCheckout, pub volume_id: VolumeId, @@ -231,6 +235,56 @@ pub fn local_options(root: impl Into) -> LocalOptions { } impl Store { + /// Reconciles an SDK head after crash recovery completed a host tree swap. + pub async fn recover_workspace_head( + &mut self, + recovered: &crate::rewind::RecoveredSwap, + ) -> Result<()> { + if !recovered.published { + return Ok(()); + } + let current = self + .workspace + .head() + .await + .map_err(EngineError::fs("workspace head"))?; + if current.id() != recovered.target { + let target = self + .workspace + .generation(recovered.target) + .await + .map_err(EngineError::fs("recover rewind generation"))?; + let key = acyclic_fs::IdempotencyKey::from_bytes( + recovered.target.digest().as_bytes()[..16] + .try_into() + .map_err(|_| EngineError::Store("invalid generation digest".into()))?, + ); + match self + .workspace + .restore_generation(&target, current.id(), key) + .await + .map_err(EngineError::fs("recover workspace rewind"))? + { + WorkspaceRestore::Restored(_) + | WorkspaceRestore::AlreadyRestored(_) + | WorkspaceRestore::Current(_) => {} + WorkspaceRestore::Stale(_) + | WorkspaceRestore::Fenced + | WorkspaceRestore::IdempotencyConflict => { + return Err(EngineError::Store( + "workspace head changed during rewind recovery".into(), + )); + } + } + } + self.checkout = self + .workspace + .checkout(GenerationSelector::Head, writable_head()) + .await + .map_err(EngineError::fs("refresh recovered workspace"))?; + Ok(()) + } + /// Creates the store for a repo: directories, volume, meta record. /// Fails if the store already exists. pub async fn init(repo_root: &Path, paths: StorePaths) -> Result { @@ -267,6 +321,10 @@ impl Store { .await .map_err(EngineError::fs("checkout head"))? .value; + let workspace = fs + .open_volume_workspace("main", volume_id) + .await + .map_err(EngineError::fs("adopt workspace"))?; let repo_root = repo_root.canonicalize()?; let meta = StoreMeta { @@ -277,6 +335,7 @@ impl Store { atomic_write_json(&paths.meta(), &meta)?; Ok(Self { fs, + workspace, volume, checkout, volume_id, @@ -334,6 +393,10 @@ impl Store { .await .map_err(EngineError::fs("checkout head"))? .value; + let workspace = fs + .open_volume_workspace("main", meta.volume_id) + .await + .map_err(EngineError::fs("adopt workspace"))?; crate::trace!( "store", "open: object store {objects_ms:.1}ms, volume {volume_ms:.1}ms, head checkout {:.1}ms", @@ -341,6 +404,7 @@ impl Store { ); Ok(Self { fs, + workspace, volume, checkout, volume_id: meta.volume_id, @@ -354,18 +418,10 @@ impl Store { &self, generation: acyclic_fs::GenerationId, ) -> Result { - let cancel = CancellationToken::new(); - Ok(self - .volume - .checkout( - GenerationSelector::Exact(generation), - read_only(), - WorkCounters::UNBOUNDED, - &cancel, - ) + self.workspace + .checkout(GenerationSelector::Exact(generation), read_only()) .await - .map_err(EngineError::fs("checkout exact"))? - .value) + .map_err(EngineError::fs("checkout exact")) } } From 323bbc6f63a6f299101ed409d19e237a7c4a584c Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 08:43:53 +0100 Subject: [PATCH 044/106] Use workspace facade for plugin checkouts --- crates/acyclic/src/pipeline.rs | 21 ++++++-------------- crates/acyclic/src/store.rs | 35 ++++++++++------------------------ 2 files changed, 16 insertions(+), 40 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 79b64ca..6af7654 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -1928,16 +1928,13 @@ impl Pipeline { async fn scratch_checkout(&mut self, base: GenerationId) -> Result> { let checkout = self .store - .volume + .workspace .checkout( acyclic_fs::model::GenerationSelector::Exact(base), crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, ) .await - .map_err(EngineError::fs("scratch checkout"))? - .value; + .map_err(EngineError::fs("scratch checkout"))?; Ok(Arc::new(SharedLocalCheckout::with_publication( checkout, MountPublication::Manual, @@ -1990,16 +1987,13 @@ impl Pipeline { // front of a fork. let checkout = self .store - .volume + .workspace .checkout( acyclic_fs::model::GenerationSelector::Exact(base), crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, ) .await - .map_err(EngineError::fs("fork checkout"))? - .value; + .map_err(EngineError::fs("fork checkout"))?; let config = checkout.volume_config(); let volume_id = checkout.volume_id(); Ok(ForkSeed { @@ -2093,16 +2087,13 @@ impl Pipeline { self.state = State::Rewinding; self.store.checkout = self .store - .volume + .workspace .checkout( acyclic_fs::model::GenerationSelector::Head, crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, ) .await - .map_err(EngineError::fs("refresh checkout"))? - .value; + .map_err(EngineError::fs("refresh checkout"))?; let row = self.index.record( generation, CheckpointKind::Manual, diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index c3150b3..bd9024f 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -22,8 +22,6 @@ use crate::{EngineError, Result}; /// Concrete checkout type for the local backend. pub type LocalCheckout = Checkout; -/// Concrete volume type for the local backend. -pub type LocalVolume = acyclic_fs::LocalVolume; /// Concrete workspace type for the local backend. pub type LocalWorkspace = acyclic_fs::Workspace; @@ -181,7 +179,6 @@ pub struct StoreMeta { pub struct Store { pub fs: LocalFs, pub workspace: LocalWorkspace, - pub volume: LocalVolume, pub checkout: LocalCheckout, pub volume_id: VolumeId, pub paths: StorePaths, @@ -311,20 +308,15 @@ impl Store { .await .map_err(EngineError::fs("create volume"))? .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - writable_head(), - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("checkout head"))? - .value; let workspace = fs .open_volume_workspace("main", volume_id) .await .map_err(EngineError::fs("adopt workspace"))?; + let checkout = workspace + .checkout(GenerationSelector::Head, writable_head()) + .await + .map_err(EngineError::fs("checkout head"))?; + drop(volume); let repo_root = repo_root.canonicalize()?; let meta = StoreMeta { @@ -336,7 +328,6 @@ impl Store { Ok(Self { fs, workspace, - volume, checkout, volume_id, paths, @@ -383,20 +374,15 @@ impl Store { .value; let volume_ms = crate::trace::ms(phase); let phase = std::time::Instant::now(); - let checkout = volume - .checkout( - GenerationSelector::Head, - writable_head(), - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("checkout head"))? - .value; let workspace = fs .open_volume_workspace("main", meta.volume_id) .await .map_err(EngineError::fs("adopt workspace"))?; + let checkout = workspace + .checkout(GenerationSelector::Head, writable_head()) + .await + .map_err(EngineError::fs("checkout head"))?; + drop(volume); crate::trace!( "store", "open: object store {objects_ms:.1}ms, volume {volume_ms:.1}ms, head checkout {:.1}ms", @@ -405,7 +391,6 @@ impl Store { Ok(Self { fs, workspace, - volume, checkout, volume_id: meta.volume_id, paths, From 5340d899e7b227b139f8abf9d5dad6c79f203051 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 08:45:36 +0100 Subject: [PATCH 045/106] Avoid reopening adopted plugin volumes --- crates/acyclic/src/store.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index bd9024f..f30ee49 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -309,14 +309,12 @@ impl Store { .map_err(EngineError::fs("create volume"))? .value; let workspace = fs - .open_volume_workspace("main", volume_id) - .await + .adopt_volume_workspace("main", volume) .map_err(EngineError::fs("adopt workspace"))?; let checkout = workspace .checkout(GenerationSelector::Head, writable_head()) .await .map_err(EngineError::fs("checkout head"))?; - drop(volume); let repo_root = repo_root.canonicalize()?; let meta = StoreMeta { @@ -375,14 +373,12 @@ impl Store { let volume_ms = crate::trace::ms(phase); let phase = std::time::Instant::now(); let workspace = fs - .open_volume_workspace("main", meta.volume_id) - .await + .adopt_volume_workspace("main", volume) .map_err(EngineError::fs("adopt workspace"))?; let checkout = workspace .checkout(GenerationSelector::Head, writable_head()) .await .map_err(EngineError::fs("checkout head"))?; - drop(volume); crate::trace!( "store", "open: object store {objects_ms:.1}ms, volume {volume_ms:.1}ms, head checkout {:.1}ms", From 3a5a4bfb6632289ed27ca34ab0e252193fe29ed4 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 08:49:36 +0100 Subject: [PATCH 046/106] Observe daemon readiness without fixed delay --- crates/acyclic/src/client.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index cde39ba..cbc40b4 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -488,6 +488,7 @@ fn wait_for_socket( let started = Instant::now(); let deadline = bound.unwrap_or(Duration::from_secs(30 * 60)); let mut reported = false; + let mut retry_delay = Duration::from_millis(1); loop { if let Ok(stream) = ClientStream::connect(socket) { if let Ok(mut client) = Client::from_stream(stream) { @@ -534,6 +535,7 @@ fn wait_for_socket( eprintln!("{NAME}: daemon starting (building the first snapshot of the tree)..."); reported = true; } - std::thread::sleep(Duration::from_millis(200)); + std::thread::sleep(retry_delay); + retry_delay = (retry_delay * 2).min(Duration::from_millis(25)); } } From d97195722477412e19e615e342e2f5e91088be4d Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 08:50:21 +0100 Subject: [PATCH 047/106] Document the single plugin crate --- docs/design/02-timeline.md | 2 +- docs/design/implementation-merge.md | 10 +++++----- docs/design/implementation-rewind.md | 8 +++----- docs/windows-verification.md | 12 ++++++------ 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/design/02-timeline.md b/docs/design/02-timeline.md index a132ce4..3a98b2a 100644 --- a/docs/design/02-timeline.md +++ b/docs/design/02-timeline.md @@ -35,7 +35,7 @@ Same engine, adds the time dimension. Fast-follow; bundles into Launch 1 if it l ## Status (2026-09-06): shipped -Implemented in `acyclic-engine` (index), the daemon, the CLI, and the Claude +Implemented in `acyclic` (index), the daemon, the CLI, and the Claude Code adapter; `tests/acceptance/timeline.sh` covers all four acceptance criteria and runs in `run-all.sh`. diff --git a/docs/design/implementation-merge.md b/docs/design/implementation-merge.md index 61979db..6dc9ff5 100644 --- a/docs/design/implementation-merge.md +++ b/docs/design/implementation-merge.md @@ -165,12 +165,12 @@ consistency: the fork survives every non-landing outcome. ## Where the code goes ``` -crates/acyclic-engine/src/merge.rs NEW — entry table, merge3 port, marker scan -crates/acyclic-engine/src/pipeline.rs BuildGeneration { from, entries } request -crates/acyclic-engine/src/fork.rs ForkSeed.base becomes mutable; OpenConflict -crates/acyclic-engine/src/config.rs [merge] max_file_bytes +crates/acyclic/src/merge.rs NEW — entry table, merge3 port, marker scan +crates/acyclic/src/pipeline.rs BuildGeneration { from, entries } request +crates/acyclic/src/fork.rs ForkSeed.base becomes mutable; OpenConflict +crates/acyclic/src/config.rs [merge] max_file_bytes crates/acyclic/src/server.rs replay_onto_head → merge_onto_head; rebase path -crates/acyclic-proto/src/lib.rs PromoteInfo.merged_files; ForkInfo.conflict +crates/acyclic/src/lib.rs PromoteInfo.merged_files; ForkInfo.conflict crates/acyclic/src/main.rs promote / forks output crates/acyclic/src/install.rs skill: PARTITION + conflict resolution tests/acceptance/merge.sh G4/G5-adjacent flips; G13–G26 added diff --git a/docs/design/implementation-rewind.md b/docs/design/implementation-rewind.md index c41a534..544dddc 100644 --- a/docs/design/implementation-rewind.md +++ b/docs/design/implementation-rewind.md @@ -17,16 +17,14 @@ The engine is imported, not built: `acyclic-fs` + `acyclic-fs-mount` via path de ``` crates/ - acyclic-engine/ lib: store, pipeline, index, rewind, diff, config - acyclic/ bin: CLI + hidden `__daemon` subcommand; private protocol module - acyclic-qual/ Phase 0 harness → standing bench suite (exists) + acyclic/ one binary package: CLI, daemon, private engine and protocol modules adapters/claude-code/ hooks, /rewind command, self-rollback skill tests/acceptance/ one script per acceptance criterion ``` --- -# Phase 1 — the engine library (`acyclic-engine`) +# Phase 1 — the engine library (`acyclic`) **Goal:** everything below the wire works and survives crashes, proven by tests. @@ -123,7 +121,7 @@ Built as `acyclic hook ` + `acyclic install ` (the installer embeds Shipped: - Acceptance: `exclusions.sh` (declared paths never captured, noop on excluded-only edits, restore refused, rename into an excluded prefix scrubbed, rewind carries the live copies, pre-rule history untouched) and `growth.sh` (60 distinct checkpoints cost ~52 KB each, identical on a 4x larger tree: per-checkpoint overhead is tree pages, never a tree copy). Journey/soak/crash/latency were already green; the migration-script scenario is journey.sh's Bash side-effect step. -- Snapshot exclusions (`exclude` in config): `acyclic-engine/src/exclude.rs`. Watcher hints at or under a rule are dropped before capture; renames across the boundary re-examine the uncovered side; a capture hinted at an ancestor (or the baseline) is followed by a checkout scrub before the generation is checkpointed; a full rewind moves the live excluded paths into the new tree before the swap, journaled as a `Carrying` phase so kill -9 at any point returns them. +- Snapshot exclusions (`exclude` in config): `crates/acyclic/src/exclude.rs`. Watcher hints at or under a rule are dropped before capture; renames across the boundary re-examine the uncovered side; a capture hinted at an ancestor (or the baseline) is followed by a checkout scrub before the generation is checkpointed; a full rewind moves the live excluded paths into the new tree before the swap, journaled as a `Carrying` phase so kill -9 at any point returns them. - Retention: `status` reports store size, trash is TTL-pruned. **No fs GC, no purge in v1**, and this is a finding, not an omission: at sdk `8eced48`, `collect_local_garbage` keeps only authority heads plus retention facts (`RetentionKind::{Checkpoint,Pin,ForkBase}`), `retain_workspace_generation` is create-only (no release fact exists), and `prove_generation_closure` does not follow `GenerationRoot::parents`. Running GC would delete every checkpoint but the head; pinning every checkpoint first would make the store append-only forever. Purge cannot remove bytes from a retained generation for the same reason. **Upstream ask:** a retention-release fact (mirror of `encode_workspace_deleted` for retention authorities) honoured by the collector, plus a bulk "retain these N generations" call. With that, the plugin's policy is straightforward: pin what the TTL keeps, release the rest, collect. - Release engineering: `deny.toml` + a `deny` CI job (licenses inside a permissive allowlist, advisories, sources); an SPDX SBOM generated from `Cargo.lock` per target in `release.yml`, attested against each binary with `actions/attest-sbom`, one copy attached to the GitHub release and listed in `SHA256SUMS`; `scripts/install.sh` (verified download into `~/.local/bin`, `file://` base URL for offline tests); `scripts/install-smoke.sh` runs it on a bare Debian container and drives init → checkpoint → rewind, exclusions included. npm `@acyclic-labs/plugin` 0.0.1 is live. - Docs: README leads with the differentiators, documents `exclude`, and states the retention stance and caveats. diff --git a/docs/windows-verification.md b/docs/windows-verification.md index a7f6581..d9967e4 100644 --- a/docs/windows-verification.md +++ b/docs/windows-verification.md @@ -15,13 +15,13 @@ lint job cannot see. | Area | Change | | --- | --- | | `crates/acyclic/src/ipc.rs` | New. The transport split: Unix domain socket vs. Windows named pipe, with an owner-only pipe DACL. | -| `crates/acyclic-engine/src/names.rs` | New. The one definition of how a host name becomes engine bytes. | +| `crates/acyclic/src/names.rs` | New. The one definition of how a host name becomes engine bytes. | | `crates/acyclic/src/client.rs` | `ipc::ClientStream`; daemon spawn no longer leaks stdio handles or stands in the repo. | | `crates/acyclic/src/server.rs` | `ipc::Listener`/`ipc::ServerStream`; fork route names carry the host encoding. | -| `crates/acyclic-engine/src/store.rs` | Volume profile and component byte budget come from `names`. | -| `crates/acyclic-engine/src/rewind.rs` | Windows directory exchange; journal write fixed; staging is retry-safe. | -| `crates/acyclic-engine/src/guard.rs` | Guarded prefixes and `AppleDouble` matching in the host encoding. | -| `crates/acyclic-engine/src/{diff,exclude}.rs` | Name decoding via `names` instead of per-file UTF-8 fallbacks. | +| `crates/acyclic/src/store.rs` | Volume profile and component byte budget come from `names`. | +| `crates/acyclic/src/rewind.rs` | Windows directory exchange; journal write fixed; staging is retry-safe. | +| `crates/acyclic/src/guard.rs` | Guarded prefixes and `AppleDouble` matching in the host encoding. | +| `crates/acyclic/src/{diff,exclude}.rs` | Name decoding via `names` instead of per-file UTF-8 fallbacks. | | `crates/acyclic/src/fork.rs` | Windows forks require Projected File System. | | `crates/acyclic/src/main.rs` | The client steps out of the tree before asking for a whole-tree swap. | | `.github/workflows/{ci,release}.yml` | A `windows` CI job; `win32/x64` release matrix entry. | @@ -53,7 +53,7 @@ name that arrives in any other encoding is *projected as mojibake rather than rejected* — fork route ids passed as raw ASCII came back as six garbage characters. Guarded fork paths have the sharper version of the same problem: they compare configured prefixes byte-for-byte against mount path components, -and a guard that never matches **fails open**. `crates/acyclic-engine/src/names.rs` +and a guard that never matches **fails open**. `crates/acyclic/src/names.rs` exists so there is exactly one place this can be got wrong. Because the profile is fixed for the life of a volume, a store created on From 25d27fd0b6ee742ba1d83253df20093058774e7b Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 08:59:25 +0100 Subject: [PATCH 048/106] Keep restart rewind fixture protocol-valid --- crates/acyclic/tests/restart_rewind.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/acyclic/tests/restart_rewind.rs b/crates/acyclic/tests/restart_rewind.rs index a9c7683..645df48 100644 --- a/crates/acyclic/tests/restart_rewind.rs +++ b/crates/acyclic/tests/restart_rewind.rs @@ -20,7 +20,7 @@ fn cli_recovers_a_missing_repo_before_loading_config() -> Result<(), Box Date: Sat, 19 Sep 2026 09:33:09 +0100 Subject: [PATCH 049/106] Remove deferred startup sweep from production --- crates/acyclic/src/spec_runner.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/acyclic/src/spec_runner.rs b/crates/acyclic/src/spec_runner.rs index 335f90b..7174ca0 100644 --- a/crates/acyclic/src/spec_runner.rs +++ b/crates/acyclic/src/spec_runner.rs @@ -24,7 +24,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; // Only the Unix sweep names the product; on Windows there is no sweep. -#[cfg(unix)] +#[cfg(all(unix, test))] use acyclic::product::NAME; use acyclic::spec::RunOutcome; use tokio::io::AsyncWriteExt; @@ -403,7 +403,7 @@ fn sweep_stale_runs(spec_runs: &Path) { /// that dies — however it dies — takes the run's whole tree with it. /// Whether the live process really is the run we recorded, rather than /// whatever inherited its pid. -#[cfg(unix)] +#[cfg(all(unix, test))] fn command_name_matches(pid: i32, expected: &str) -> bool { let Ok(output) = std::process::Command::new("ps") .args(["-o", "comm=", "-p", &pid.to_string()]) From ee655063177b52e470a7088f533001ce7ac18cac Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 09:38:06 +0100 Subject: [PATCH 050/106] Keep idle startup constant in repository size --- crates/acyclic/src/pipeline.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 6af7654..5b28e42 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -1595,7 +1595,10 @@ impl Pipeline { if self.config.auto_checkpoint_idle_ms == 0 { return; } - if self.state == State::NeedsBaseline && self.ensure_ready().await.is_err() { + // An idle daemon must remain O(1) in repository size. The first + // consumer operation that needs authenticated contents performs the + // baseline; a timer alone is not such an operation. + if self.state == State::NeedsBaseline { return; } if self.state != State::Ready { From f5278c6e19d48749ff4997e921482b78aac0fdd1 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 09:42:35 +0100 Subject: [PATCH 051/106] Keep Unix sweep dependencies with test coverage --- crates/acyclic/src/spec_runner.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/acyclic/src/spec_runner.rs b/crates/acyclic/src/spec_runner.rs index 7174ca0..335f90b 100644 --- a/crates/acyclic/src/spec_runner.rs +++ b/crates/acyclic/src/spec_runner.rs @@ -24,7 +24,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; // Only the Unix sweep names the product; on Windows there is no sweep. -#[cfg(all(unix, test))] +#[cfg(unix)] use acyclic::product::NAME; use acyclic::spec::RunOutcome; use tokio::io::AsyncWriteExt; @@ -403,7 +403,7 @@ fn sweep_stale_runs(spec_runs: &Path) { /// that dies — however it dies — takes the run's whole tree with it. /// Whether the live process really is the run we recorded, rather than /// whatever inherited its pid. -#[cfg(all(unix, test))] +#[cfg(unix)] fn command_name_matches(pid: i32, expected: &str) -> bool { let Ok(output) = std::process::Command::new("ps") .args(["-o", "comm=", "-p", &pid.to_string()]) From d76d251f8669b27fdfc3938d3ff8c0d915b42e3a Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 09:44:48 +0100 Subject: [PATCH 052/106] Activate auto checkpointing after first real use --- crates/acyclic/tests/pipeline.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index 1cb7f94..3dbb92e 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -160,9 +160,8 @@ fn checkpoint_rewind_journey() { assert!(by_name.contains(&(PathBuf::from(".env"), ChangeKind::Removed))); } -/// The safety net for hosts with no lifecycle-hook API (Claude Desktop over -/// MCP): an edit with no `handle.checkpoint()` call at all still gets -/// checkpointed once the idle timer fires. +/// Once a consumer establishes the authenticated baseline, the safety net for +/// hosts with no lifecycle-hook API still checkpoints later unannounced edits. #[test] fn idle_timer_auto_checkpoints_changes_no_host_asked_for() { let repo = tempfile::tempdir().expect("repo"); @@ -188,19 +187,13 @@ fn idle_timer_auto_checkpoints_changes_no_host_asked_for() { let (handle, thread) = pipeline::spawn(store, index, config); runtime.block_on(async { - // Wait for the enabled background observer to establish its baseline - // before editing; status itself remains scan free. - let deadline = tokio::time::Instant::now() + Duration::from_secs(15); - let baseline_row = loop { - if let Some(row) = handle.status().await.expect("status").last_checkpoint { - break Some(row); - } - assert!( - tokio::time::Instant::now() < deadline, - "baseline never completed" - ); - tokio::time::sleep(Duration::from_millis(25)).await; - }; + let baseline_row = Some( + handle + .checkpoint(CheckpointKind::Baseline, Attribution::default()) + .await + .expect("baseline") + .row_id, + ); // No hook, no explicit checkpoint call — just an edit, like a host // with no lifecycle-hook API would produce. @@ -364,7 +357,10 @@ fn periodic_requests_do_not_postpone_the_auto_checkpoint() { let (handle, thread) = pipeline::spawn(store, index, config); runtime.block_on(async { - handle.status().await.expect("status"); + handle + .checkpoint(CheckpointKind::Baseline, Attribution::default()) + .await + .expect("baseline"); std::fs::write(repo.path().join("a.txt"), b"two\n").expect("edit"); // Poll far more often than the idle interval, for far longer. for _ in 0..40 { From a4223b32d2186f1788bbe89792a552a1def06014 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 09:47:06 +0100 Subject: [PATCH 053/106] Compile deferred Unix cleanup only in its test --- crates/acyclic/src/spec_runner.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/acyclic/src/spec_runner.rs b/crates/acyclic/src/spec_runner.rs index 335f90b..a55676a 100644 --- a/crates/acyclic/src/spec_runner.rs +++ b/crates/acyclic/src/spec_runner.rs @@ -24,7 +24,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; // Only the Unix sweep names the product; on Windows there is no sweep. -#[cfg(unix)] +#[cfg(all(unix, test))] use acyclic::product::NAME; use acyclic::spec::RunOutcome; use tokio::io::AsyncWriteExt; @@ -356,7 +356,7 @@ impl Drop for OwnedJob { /// Matched on the recorded command name as well as the pid, so a reused pid /// belonging to something else is never signalled — the check costs one /// `ps` and removes the whole class of mistake. -#[cfg(unix)] +#[cfg(all(unix, test))] #[allow( unsafe_code, reason = "kill(pid, 0) only tests for the process's existence; the \ @@ -403,7 +403,7 @@ fn sweep_stale_runs(spec_runs: &Path) { /// that dies — however it dies — takes the run's whole tree with it. /// Whether the live process really is the run we recorded, rather than /// whatever inherited its pid. -#[cfg(unix)] +#[cfg(all(unix, test))] fn command_name_matches(pid: i32, expected: &str) -> bool { let Ok(output) = std::process::Command::new("ps") .args(["-o", "comm=", "-p", &pid.to_string()]) From 8ee12b5352858fefe6d5597ca071a304c4b7f739 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 09:48:55 +0100 Subject: [PATCH 054/106] Describe lazy daemon readiness accurately --- crates/acyclic/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index 19ce54b..3335e41 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -547,14 +547,15 @@ fn init(repo: &Path) -> i32 { .map_err(|error| error.to_string())?; println!("store created at {}", paths.root.display()); } - // Spawning the daemon builds (or refreshes) the baseline. + // Spawning opens metadata and the watcher without scanning descendants. + // The first content-dependent operation establishes the baseline. let mut client = connect(repo, Spawn::Allowed).map_err(|error| match error { ConnectError::NoDaemon => "daemon failed to start".to_owned(), ConnectError::Starting => "daemon is still starting".to_owned(), ConnectError::Other(message) => message, })?; client.call(proto::Op::Ping)?; - println!("daemon ready — checkpointing is on"); + println!("daemon ready — checkpointing activates on first use"); print_mount_capability(); Ok(()) })(); From 73c305bd775f5ce08a19675be63e52f6ad2ec152 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 09:53:26 +0100 Subject: [PATCH 055/106] Prove idle startup remains scan free --- crates/acyclic/src/server.rs | 2 +- crates/acyclic/tests/pipeline.rs | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 5dbfd02..db77514 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -189,7 +189,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { let index = Index::open(&paths.index_db()).map_err(|error| error.to_string())?; lap("index open"); let (handle, pipeline_thread) = pipeline::spawn(store, index, config.clone()); - lap("pipeline thread spawn (baseline runs on it)"); + lap("pipeline metadata thread spawn"); let shutdown = Arc::new(Notify::new()); let mounts = fork::mount_capability(); diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index 3dbb92e..d2e90aa 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -54,6 +54,44 @@ fn startup_failure_is_reported_by_status() { thread.join().expect("pipeline thread"); } +/// Starting the daemon and polling metadata must stay constant in repository +/// size. Even an enabled idle timer cannot authenticate content before a real +/// content operation asks for it. +#[test] +fn idle_metadata_traffic_never_establishes_the_baseline() { + let repo = tempfile::tempdir().expect("repo"); + let stores = tempfile::tempdir().expect("stores"); + std::fs::write(repo.path().join("a.txt"), b"one\n").expect("seed"); + + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("paths"); + let runtime = tokio::runtime::Runtime::new().expect("runtime"); + let store = runtime + .block_on(Store::init(repo.path(), paths.clone())) + .expect("init store"); + let index = Index::open(&paths.index_db()).expect("index"); + let config = Config { + auto_checkpoint_idle_ms: 20, + ..fast_config() + }; + let (handle, thread) = pipeline::spawn(store, index, config); + + runtime.block_on(async { + let deadline = tokio::time::Instant::now() + Duration::from_millis(150); + while tokio::time::Instant::now() < deadline { + let status = handle.status().await.expect("status"); + assert_eq!(status.state, pipeline::State::NeedsBaseline); + assert_eq!(status.last_checkpoint, None); + tokio::time::sleep(Duration::from_millis(10)).await; + } + handle.shutdown().await.expect("shutdown"); + }); + thread.join().expect("pipeline thread"); + assert!(read_only_index(&paths.index_db()) + .latest() + .expect("query") + .is_none()); +} + #[test] fn checkpoint_rewind_journey() { let repo = tempfile::tempdir().expect("repo"); From 72f19780e1a0460b676bae3621235de3bbad6602 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 10:33:14 +0100 Subject: [PATCH 056/106] Open native watcher on first filesystem demand --- crates/acyclic/src/pipeline.rs | 73 +++++++++++++++++--------------- crates/acyclic/tests/pipeline.rs | 19 ++++++--- 2 files changed, 51 insertions(+), 41 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 5b28e42..33b3148 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -835,54 +835,28 @@ impl Pipeline { let cancel = CancellationToken::new(); let repo_root: PathBuf = store.repo_root.clone(); - let mut watch = NativeWatch::open( - &repo_root, - NativeWatchOptions { - limits: VolumeLimits::default(), - maximum_queued_changes: WATCH_QUEUE, - recursive: true, - }, - ) - .map_err(EngineError::fs("open watcher"))?; - #[cfg(target_os = "windows")] - let admitted = Self::admit_continuity(&store, &index, &mut watch)?; - #[cfg(not(target_os = "windows"))] - let admitted = false; - if !admitted { - watch - .begin_rescan() - .map_err(EngineError::fs("begin rescan"))?; - } - let options = CaptureOptions { - source_root: repo_root.clone(), - expected_root_identity: capture_root_identity(&repo_root) - .map_err(EngineError::fs("root identity"))?, + source_root: repo_root, + // First filesystem demand replaces this before any capture. Keeping + // startup free of native root access lets metadata-only consumers + // run even when the repository is temporarily unavailable. + expected_root_identity: acyclic_fs::NativeRootIdentity::from_bytes([0; 16]), maximum_paths: MAXIMUM_CAPTURE_PATHS, maximum_extent_spans: MAXIMUM_EXTENT_SPANS, }; let exclusions = Exclusions::parse(&config.exclude)?; - let latest = admitted.then(|| index.latest()).transpose()?.flatten(); let pipeline = Self { store, index, config, - watch: Some(watch), + watch: None, options, exclusions, cancel, - state: if admitted { - State::Ready - } else { - State::NeedsBaseline - }, - last_generation: latest - .as_ref() - .map_or(GenerationId::new(acyclic_fs::Digest::ZERO), |row| { - row.generation - }), - last_checkpoint_row: latest.map(|row| row.id), + state: State::NeedsBaseline, + last_generation: GenerationId::new(acyclic_fs::Digest::ZERO), + last_checkpoint_row: None, checkpoints_since_commit: 0, last_activity: Instant::now(), watcher_health: WatcherHealth::default(), @@ -923,6 +897,35 @@ impl Pipeline { if self.state == State::Ready { return Ok(()); } + if self.watch.is_none() { + let repo_root = self.store.repo_root.clone(); + self.options.expected_root_identity = + capture_root_identity(&repo_root).map_err(EngineError::fs("root identity"))?; + let mut watch = NativeWatch::open( + &repo_root, + NativeWatchOptions { + limits: VolumeLimits::default(), + maximum_queued_changes: WATCH_QUEUE, + recursive: true, + }, + ) + .map_err(EngineError::fs("open watcher"))?; + #[cfg(target_os = "windows")] + if Self::admit_continuity(&self.store, &self.index, &mut watch)? { + let latest = self.index.latest()?.ok_or_else(|| { + EngineError::Store("continuity admitted without an index row".into()) + })?; + self.last_generation = latest.generation; + self.last_checkpoint_row = Some(latest.id); + self.watch = Some(watch); + self.state = State::Ready; + return Ok(()); + } + watch + .begin_rescan() + .map_err(EngineError::fs("begin rescan"))?; + self.watch = Some(watch); + } self.baseline(CheckpointKind::Baseline).await } diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index d2e90aa..ac944d6 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -34,7 +34,7 @@ fn fast_config() -> Config { } #[test] -fn startup_failure_is_reported_by_status() { +fn native_startup_failure_is_deferred_until_filesystem_demand() { let repo = tempfile::tempdir().expect("repo"); let stores = tempfile::tempdir().expect("stores"); let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("paths"); @@ -46,11 +46,15 @@ fn startup_failure_is_reported_by_status() { std::fs::remove_dir(repo.path()).expect("remove empty repo before watcher opens"); let (handle, thread) = pipeline::spawn(store, index, fast_config()); - let error = runtime + let status = runtime .block_on(handle.status()) - .expect_err("startup must fail"); - assert!(error.to_string().contains("pipeline failed to start")); - drop(handle); + .expect("metadata-only startup remains available"); + assert_eq!(status.state, pipeline::State::NeedsBaseline); + let error = runtime + .block_on(handle.checkpoint(CheckpointKind::Manual, Attribution::default())) + .expect_err("filesystem demand must fail"); + assert!(error.to_string().contains("root identity")); + runtime.block_on(handle.shutdown()).expect("shutdown"); thread.join().expect("pipeline thread"); } @@ -348,7 +352,10 @@ fn requested_checkpoint_records_changes_an_idle_tick_already_drained() { let (handle, thread) = pipeline::spawn(store, index, config); let outcome = runtime.block_on(async { - handle.status().await.expect("status"); + handle + .checkpoint(CheckpointKind::Manual, Attribution::default()) + .await + .expect("establish baseline"); std::fs::write(repo.path().join("a.txt"), b"two\n").expect("edit"); tokio::time::sleep(Duration::from_millis(600)).await; let outcome = handle From e33869e51e3b414df49ec86e11db7cdbd86eec28 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 11:11:03 +0100 Subject: [PATCH 057/106] Use SDK change sets for plugin diffs --- crates/acyclic/src/diff.rs | 84 +++++++++++++++++++------------------ crates/acyclic/src/store.rs | 31 +++++++++++--- 2 files changed, 68 insertions(+), 47 deletions(-) diff --git a/crates/acyclic/src/diff.rs b/crates/acyclic/src/diff.rs index ab04523..f344ab7 100644 --- a/crates/acyclic/src/diff.rs +++ b/crates/acyclic/src/diff.rs @@ -1,10 +1,8 @@ //! Blast-radius diff between two generations, keyed by path. //! -//! `Volume::diff_generations` returns FileId-keyed changes with no path -//! strings, so v1 walks both generations' directory records and compares -//! records per path. Content addressing makes the comparison exact: equal -//! payload object ids mean equal content. (Merkle-guided walking that skips -//! identical subtrees needs an upstream cursor/path API — tracked.) +//! Ordinary diffs use the SDK's Merkle-aware change set and resolve only +//! changed directory identities to paths. The complete summary walker remains +//! temporarily for three-way merge planning. use std::collections::BTreeMap; use std::path::PathBuf; @@ -88,43 +86,47 @@ pub async fn diff( if before == after { return Ok(Vec::new()); } - let mut before_checkout = store.checkout_exact(before).await?; - let mut after_checkout = store.checkout_exact(after).await?; - let before_map = walk(&mut before_checkout).await?; - let after_map = walk(&mut after_checkout).await?; - - let mut changes = Vec::new(); - for (path, summary) in &before_map { - match after_map.get(path) { - None => changes.push(FileChange { - path: path.clone(), - change: ChangeKind::Removed, - file_kind: summary.kind, - }), - Some(other) if other == summary => {} - Some(other) => { - let change = if other.kind == summary.kind && other.payload == summary.payload { - ChangeKind::MetadataOnly - } else { - ChangeKind::Modified - }; - changes.push(FileChange { - path: path.clone(), - change, - file_kind: other.kind, + let before = store.generation(before).await?; + let after = store.generation(after).await?; + let set = before + .diff_to(&after, u32::MAX) + .await + .map_err(EngineError::fs("diff generations"))?; + let paths = set + .changed_paths(u32::MAX) + .await + .map_err(EngineError::fs("resolve changed paths"))?; + let mut changes = paths + .into_iter() + .filter_map(|path| { + let (change, file_kind) = match (&path.before, &path.after) { + (None, Some(after)) => (ChangeKind::Added, after.kind), + (Some(before), None) => (ChangeKind::Removed, before.kind), + (Some(before), Some(after)) => { + let change = if before.kind == after.kind && before.payload == after.payload { + ChangeKind::MetadataOnly + } else { + ChangeKind::Modified + }; + (change, after.kind) + } + (None, None) => return None, + }; + let path = path + .path + .components() + .iter() + .fold(PathBuf::new(), |mut path, component| { + path.push(crate::names::bytes_to_os(component.as_bytes())); + path }); - } - } - } - for (path, summary) in &after_map { - if !before_map.contains_key(path) { - changes.push(FileChange { - path: path.clone(), - change: ChangeKind::Added, - file_kind: summary.kind, - }); - } - } + Some(FileChange { + path, + change, + file_kind, + }) + }) + .collect::>(); // Snapshots carry `.git` so rewind restores it, but a blast-radius // report is about the working tree: object and ref churn from ordinary // git commands would otherwise swamp the real changes. diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index f30ee49..05ae9d4 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -24,6 +24,8 @@ use crate::{EngineError, Result}; pub type LocalCheckout = Checkout; /// Concrete workspace type for the local backend. pub type LocalWorkspace = acyclic_fs::Workspace; +/// Concrete immutable generation type for the local backend. +pub type LocalGeneration = acyclic_fs::Generation; /// Batch limits sized for large monorepos: the fs defaults (2,048) reject any /// baseline capture beyond ~2k paths. Immutable per volume — size generously. @@ -220,14 +222,20 @@ pub fn read_only() -> CheckoutMode { } } -/// Barrier durability for both providers: a full device flush per journal -/// frame costs ~5ms each on Apple SSDs and a small capture issues dozens, -/// while the store only needs to survive a daemon crash — a torn tail after -/// power loss just drops the newest checkpoint. +/// Exact local durability supported by this platform. pub fn local_options(root: impl Into) -> LocalOptions { let mut options = LocalOptions::new(root); - options.stream.durability = LocalStreamDurability::Barrier; - options.objects.durability = LocalObjectsDurability::Barrier; + #[cfg(target_vendor = "apple")] + { + // Apple exposes an ordered barrier that does not drain the device cache. + options.stream.durability = LocalStreamDurability::Barrier; + options.objects.durability = LocalObjectsDurability::Barrier; + } + #[cfg(not(target_vendor = "apple"))] + { + options.stream.durability = LocalStreamDurability::FullFlush; + options.objects.durability = LocalObjectsDurability::FullFlush; + } options } @@ -404,6 +412,17 @@ impl Store { .await .map_err(EngineError::fs("checkout exact")) } + + /// Opens one authenticated immutable generation handle. + pub async fn generation( + &self, + generation: acyclic_fs::GenerationId, + ) -> Result { + self.workspace + .generation(generation) + .await + .map_err(EngineError::fs("open exact generation")) + } } /// Short per-user directory for daemon sockets. Created 0700 on first use. From c036931707c527c10340ec059a74db0a56491651 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 11:19:46 +0100 Subject: [PATCH 058/106] Make merge planning proportional to changed paths --- crates/acyclic/src/diff.rs | 102 ++---------------------------- crates/acyclic/src/merge.rs | 123 ++++++++++++++++++++++++++++-------- 2 files changed, 99 insertions(+), 126 deletions(-) diff --git a/crates/acyclic/src/diff.rs b/crates/acyclic/src/diff.rs index f344ab7..d2240b8 100644 --- a/crates/acyclic/src/diff.rs +++ b/crates/acyclic/src/diff.rs @@ -1,21 +1,16 @@ //! Blast-radius diff between two generations, keyed by path. //! -//! Ordinary diffs use the SDK's Merkle-aware change set and resolve only -//! changed directory identities to paths. The complete summary walker remains -//! temporarily for three-way merge planning. +//! The SDK's Merkle-aware change set resolves only changed identities to paths. -use std::collections::BTreeMap; use std::path::PathBuf; -use acyclic_fs::kernel::{FileKind, NamespacePath}; -use acyclic_fs::{CancellationToken, GenerationId, ObjectId, WorkCounters}; +use acyclic_fs::kernel::FileKind; +use acyclic_fs::GenerationId; use serde::{Deserialize, Serialize}; -use crate::store::{LocalCheckout, Store}; +use crate::store::Store; use crate::{EngineError, Result}; -const PAGE_ENTRIES: u32 = 1_024; - /// One changed path between two generations. #[derive(Clone, Debug, Eq, PartialEq)] pub struct FileChange { @@ -61,22 +56,6 @@ impl std::fmt::Display for ChangeKind { } } -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) struct RecordSummary { - pub(crate) kind: FileKind, - /// Full payload for content comparison (inline bytes included). - /// `None` for directories: their payload changes with any descendant. - pub(crate) payload: Option, - pub(crate) metadata: ObjectId, -} - -impl RecordSummary { - /// Same kind and same content; metadata (mode, times) is ignored. - pub(crate) fn same_content(&self, other: &RecordSummary) -> bool { - self.kind == other.kind && self.payload == other.payload - } -} - /// Computes the path-keyed diff `before → after`. pub async fn diff( store: &Store, @@ -142,79 +121,6 @@ pub(crate) fn is_git_internal(path: &std::path::Path) -> bool { .is_some_and(|first| first.as_os_str() == ".git") } -/// Path → record summary of every entry in `generation` (directories included). -pub(crate) async fn summaries( - store: &Store, - generation: GenerationId, -) -> Result> { - let mut checkout = store.checkout_exact(generation).await?; - walk(&mut checkout).await -} - -/// Walks every directory record in a generation into path → record summary. -/// Directories themselves are included (metadata-only changes are visible). -async fn walk(checkout: &mut LocalCheckout) -> Result> { - let cancel = CancellationToken::new(); - // The volume's own limits, not the defaults: a store raises the - // per-component byte budget on hosts whose names cost more than one - // byte per character (see `names::maximum_component_bytes`), and a walk - // built on the default would refuse paths the store happily holds. - let limits = checkout.volume_config().limits; - let mut result = BTreeMap::new(); - // (namespace components, os path) work queue, starting at the root. - let mut queue: Vec<(Vec, PathBuf)> = - vec![(Vec::new(), PathBuf::new())]; - - while let Some((components, os_path)) = queue.pop() { - let directory = NamespacePath::new(components.clone(), limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}")))?; - let mut after = None; - loop { - let page = checkout - .list_directory_records( - &directory, - after.as_ref(), - PAGE_ENTRIES, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("list directory"))? - .value; - for entry in &page.entries { - let child_os = os_path.join(logical_to_os(&entry.name)); - let payload = if entry.record.kind == FileKind::Directory { - None - } else { - Some(entry.record.payload) - }; - result.insert( - child_os.clone(), - RecordSummary { - kind: entry.record.kind, - payload, - metadata: entry.record.metadata, - }, - ); - if entry.record.kind == FileKind::Directory { - let mut child_components = components.clone(); - child_components.push(entry.name.clone()); - queue.push((child_components, child_os)); - } - } - match page.entries.last() { - Some(last) if page.has_more => after = Some(last.name.clone()), - _ => break, - } - } - } - Ok(result) -} - -fn logical_to_os(name: &acyclic_fs::kernel::LogicalName) -> std::ffi::OsString { - crate::names::bytes_to_os(name.as_bytes()) -} - #[cfg(test)] mod tests { use super::is_git_internal; diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 522f07c..74a1abf 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -14,7 +14,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; -use acyclic_fs::kernel::{FileKind, FileMetadata, MetadataField, NamespacePath}; +use acyclic_fs::kernel::{FileKind, FileMetadata, FileRecord, MetadataField, NamespacePath}; pub use acyclic_fs::text_merge::{ conflict_hunks, has_conflict_markers, ByteConflictKind as ConflictKind, }; @@ -24,7 +24,7 @@ use acyclic_fs::text_merge::{ use acyclic_fs::{ByteRange, CancellationToken, GenerationId, WorkCounters}; use bytes::Bytes; -use crate::diff::{self, RecordSummary}; +use crate::diff; use crate::rewind::{namespace_path, validate_relative}; use crate::store::{LocalCheckout, Store}; use crate::{EngineError, Result}; @@ -237,26 +237,90 @@ fn descendants<'a>( /// Paths whose content differs between two summary maps (added, removed, /// or kind/payload changed). Metadata-only differences do not count. -fn changed_paths( - before: &BTreeMap, - after: &BTreeMap, -) -> BTreeSet { - let mut set = BTreeSet::new(); - for (path, summary) in before { - match after.get(path) { - Some(other) if other.same_content(summary) => {} - _ => { - set.insert(path.clone()); +fn host_path(path: &NamespacePath) -> PathBuf { + path.components() + .iter() + .fold(PathBuf::new(), |mut path, component| { + path.push(crate::names::bytes_to_os(component.as_bytes())); + path + }) +} + +async fn changed_paths( + before: &crate::store::LocalGeneration, + after: &crate::store::LocalGeneration, +) -> Result> { + let changes = before + .diff_to(after, u32::MAX) + .await + .map_err(EngineError::fs("diff merge generations"))? + .changed_paths(u32::MAX) + .await + .map_err(EngineError::fs("resolve merge paths"))?; + Ok(changes + .into_iter() + .filter(|change| match (change.before, change.after) { + (Some(before), Some(after)) => { + before.kind != after.kind + || (before.kind != FileKind::Directory && before.payload != after.payload) } - } - } - for path in after.keys() { - if !before.contains_key(path) { - set.insert(path.clone()); - } + _ => true, + }) + .map(|change| host_path(&change.path)) + .filter(|path| !diff::is_git_internal(path)) + .collect()) +} + +async fn records_at( + generation: &crate::store::LocalGeneration, + paths: &[PathBuf], +) -> Result> { + if paths.is_empty() { + return Ok(BTreeMap::new()); } - set.retain(|path| !diff::is_git_internal(path)); - set + let namespaces = paths + .iter() + .map(|path| namespace_of(path)) + .collect::>>()?; + let records = generation + .lookup_paths(&namespaces) + .await + .map_err(EngineError::fs("lookup merge paths"))?; + Ok(paths + .iter() + .cloned() + .zip(records) + .filter_map(|(path, record)| record.map(|record| (path, record))) + .collect()) +} + +struct MergeInputs { + base: BTreeMap, + theirs: BTreeMap, + ours: BTreeMap, + ours_changed: BTreeSet, + theirs_changed: BTreeSet, +} + +async fn merge_inputs( + store: &Store, + base: GenerationId, + theirs: GenerationId, + ours: GenerationId, +) -> Result { + let base_generation = store.generation(base).await?; + let theirs_generation = store.generation(theirs).await?; + let ours_generation = store.generation(ours).await?; + let ours_changed = changed_paths(&base_generation, &ours_generation).await?; + let theirs_changed = changed_paths(&base_generation, &theirs_generation).await?; + let paths: Vec<_> = ours_changed.union(&theirs_changed).cloned().collect(); + Ok(MergeInputs { + base: records_at(&base_generation, &paths).await?, + theirs: records_at(&theirs_generation, &paths).await?, + ours: records_at(&ours_generation, &paths).await?, + ours_changed, + theirs_changed, + }) } /// Computes the merge of fork `ours` onto mainline `theirs` from `base`. @@ -272,11 +336,14 @@ pub async fn plan( ours_name: &str, limits: &MergeLimits, ) -> Result { - let base_map = diff::summaries(store, base).await?; - let theirs_map = diff::summaries(store, theirs).await?; - let ours_map = diff::summaries(store, ours).await?; - let ours_changed = changed_paths(&base_map, &ours_map); - let theirs_changed = changed_paths(&base_map, &theirs_map); + let inputs = merge_inputs(store, base, theirs, ours).await?; + let MergeInputs { + base: base_map, + theirs: theirs_map, + ours: ours_map, + ours_changed, + theirs_changed, + } = inputs; let mut base_checkout = store.checkout_exact(base).await?; let mut theirs_checkout = store.checkout_exact(theirs).await?; @@ -307,7 +374,7 @@ pub async fn plan( let b = base_map.get(path); let h = theirs_map.get(path); let f = ours_map.get(path); - let kind = |summary: Option<&RecordSummary>| summary.map(|s| s.kind); + let kind = |record: Option<&FileRecord>| record.map(|record| record.kind); match (kind(b), kind(f), kind(h)) { // Nothing to decide here: both deleted it (or both changed it // inside a now-absent dir), or independent edits below one @@ -320,7 +387,7 @@ pub async fn plan( ) => {} // Regular files on both sides. (_, Some(FileKind::Regular), Some(FileKind::Regular)) => { - if f.and_then(|s| s.payload) == h.and_then(|s| s.payload) { + if f.map(|record| record.payload) == h.map(|record| record.payload) { continue; } let base_bytes = match kind(b) { @@ -405,7 +472,7 @@ pub async fn plan( } // Symlinks retargeted identically are fine. (_, Some(FileKind::SymbolicLink), Some(FileKind::SymbolicLink)) - if f.and_then(|s| s.payload) == h.and_then(|s| s.payload) => {} + if f.map(|record| record.payload) == h.map(|record| record.payload) => {} // Everything else is a kind clash: a file on one side and a // directory or symlink on the other, symlinks retargeted // differently, or a file that became a directory on both sides. From c2a2cc5c1ddc69bb8935dc892914347355c7e7f7 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 11:27:35 +0100 Subject: [PATCH 059/106] Use SDK exact host path conversion --- crates/acyclic/src/diff.rs | 24 ++++++++++-------------- crates/acyclic/src/merge.rs | 25 ++++++++++--------------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/crates/acyclic/src/diff.rs b/crates/acyclic/src/diff.rs index d2240b8..89e8bae 100644 --- a/crates/acyclic/src/diff.rs +++ b/crates/acyclic/src/diff.rs @@ -91,21 +91,17 @@ pub async fn diff( } (None, None) => return None, }; - let path = path - .path - .components() - .iter() - .fold(PathBuf::new(), |mut path, component| { - path.push(crate::names::bytes_to_os(component.as_bytes())); - path - }); - Some(FileChange { - path, - change, - file_kind, - }) + Some( + acyclic_fs::namespace_to_host_path(&path.path) + .map(|path| FileChange { + path, + change, + file_kind, + }) + .map_err(EngineError::fs("resolve host path")), + ) }) - .collect::>(); + .collect::>>()?; // Snapshots carry `.git` so rewind restores it, but a blast-radius // report is about the working tree: object and ref churn from ordinary // git commands would otherwise swamp the real changes. diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 74a1abf..b0788cb 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -235,17 +235,6 @@ fn descendants<'a>( .filter(move |next| next.as_path() != path) } -/// Paths whose content differs between two summary maps (added, removed, -/// or kind/payload changed). Metadata-only differences do not count. -fn host_path(path: &NamespacePath) -> PathBuf { - path.components() - .iter() - .fold(PathBuf::new(), |mut path, component| { - path.push(crate::names::bytes_to_os(component.as_bytes())); - path - }) -} - async fn changed_paths( before: &crate::store::LocalGeneration, after: &crate::store::LocalGeneration, @@ -257,7 +246,7 @@ async fn changed_paths( .changed_paths(u32::MAX) .await .map_err(EngineError::fs("resolve merge paths"))?; - Ok(changes + changes .into_iter() .filter(|change| match (change.before, change.after) { (Some(before), Some(after)) => { @@ -266,9 +255,15 @@ async fn changed_paths( } _ => true, }) - .map(|change| host_path(&change.path)) - .filter(|path| !diff::is_git_internal(path)) - .collect()) + .map(|change| { + acyclic_fs::namespace_to_host_path(&change.path) + .map_err(EngineError::fs("resolve merge host path")) + }) + .filter(|path| match path { + Ok(path) => !diff::is_git_internal(path), + Err(_) => true, + }) + .collect() } async fn records_at( From 0f4b0c6bac6f5d44a8a035d909e323c1a99b5927 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 11:44:13 +0100 Subject: [PATCH 060/106] Centralize exact native paths in SDK --- crates/acyclic/src/exclude.rs | 130 +++++++------------------ crates/acyclic/src/guard.rs | 26 +++-- crates/acyclic/src/lib.rs | 1 - crates/acyclic/src/merge.rs | 8 +- crates/acyclic/src/names.rs | 172 --------------------------------- crates/acyclic/src/pipeline.rs | 12 +-- crates/acyclic/src/rewind.rs | 57 +++-------- crates/acyclic/src/server.rs | 17 +++- crates/acyclic/src/store.rs | 18 +++- crates/acyclic/tests/fork.rs | 28 ++---- crates/acyclic/tests/merge.rs | 64 +++++++----- 11 files changed, 148 insertions(+), 385 deletions(-) delete mode 100644 crates/acyclic/src/names.rs diff --git a/crates/acyclic/src/exclude.rs b/crates/acyclic/src/exclude.rs index bd22a7f..636c46f 100644 --- a/crates/acyclic/src/exclude.rs +++ b/crates/acyclic/src/exclude.rs @@ -18,8 +18,7 @@ //! release a retained generation, so nothing can be physically removed from //! history. That gap is documented in docs/design/implementation-rewind.md. -use std::ffi::OsString; -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; use acyclic_fs::kernel::{FileKind, LogicalName, NamespacePath}; use acyclic_fs::model::VolumeLimits; @@ -35,7 +34,7 @@ const PAGE_ENTRIES: u32 = 1_024; /// same rule. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Exclusions { - prefixes: Vec>>, + prefixes: Vec, } impl Exclusions { @@ -43,28 +42,22 @@ impl Exclusions { /// repo; an empty rule or one naming the repo root is refused, since /// excluding everything is the same as not running the engine. pub fn parse(patterns: &[String]) -> Result { - let mut prefixes: Vec>> = Vec::new(); + let config = crate::store::volume_config(); + let mut prefixes = Vec::new(); for pattern in patterns { let trimmed = pattern.trim().trim_end_matches('/'); - let mut components = Vec::new(); - for component in Path::new(trimmed).components() { - match component { - Component::Normal(name) => components.push(os_to_bytes(name)), - Component::CurDir => {} - _ => { - return Err(EngineError::Config(format!( - "exclude rule {pattern:?} must be a relative path inside the repo" - ))) - } - } - } - if components.is_empty() { - return Err(EngineError::Config(format!( - "exclude rule {pattern:?} would exclude the whole repo" - ))); - } - if !prefixes.contains(&components) { - prefixes.push(components); + let prefix = acyclic_fs::host_path_to_namespace( + Path::new(trimmed), + config.profile, + config.limits, + ) + .map_err(|_| { + EngineError::Config(format!( + "exclude rule {pattern:?} must be a non-empty relative path inside the repo" + )) + })?; + if !prefixes.contains(&prefix) { + prefixes.push(prefix); } } Ok(Self { prefixes }) @@ -78,51 +71,29 @@ impl Exclusions { pub fn host_paths(&self) -> Vec { self.prefixes .iter() - .map(|prefix| host_path(prefix)) + .filter_map(|prefix| acyclic_fs::namespace_to_host_path(prefix).ok()) .collect() } /// True when `relative` is an excluded path or lies under one. pub fn covers_host(&self, relative: &Path) -> bool { - let mut components = Vec::new(); - for component in relative.components() { - match component { - Component::Normal(name) => components.push(os_to_bytes(name)), - Component::CurDir => {} - _ => return false, - } - } - self.covers_bytes(&components) + let config = crate::store::volume_config(); + acyclic_fs::host_path_to_namespace(relative, config.profile, config.limits) + .is_ok_and(|path| self.covers(&path)) } /// True when `path` is an excluded path or lies under one. pub fn covers(&self, path: &NamespacePath) -> bool { - let components: Vec> = path - .components() - .iter() - .map(|name| name.as_bytes().to_vec()) - .collect(); - self.covers_bytes(&components) - } - - fn covers_bytes(&self, components: &[Vec]) -> bool { - self.prefixes - .iter() - .any(|prefix| components.starts_with(prefix)) + self.prefixes.iter().any(|prefix| path.is_within(prefix)) } /// True when `path` is a strict ancestor of an excluded path (the repo /// root included). A capture hinted at such a path may re-walk the /// excluded subtree, so the checkout needs a scrub afterwards. fn is_ancestor(&self, path: &NamespacePath) -> bool { - let components = path.components(); - self.prefixes.iter().any(|prefix| { - components.len() < prefix.len() - && components - .iter() - .zip(prefix.iter()) - .all(|(name, want)| name.as_bytes() == want.as_slice()) - }) + self.prefixes + .iter() + .any(|prefix| path.depth() < prefix.depth() && prefix.is_within(path)) } /// Drops hints the capture must not read and rewrites renames that @@ -189,14 +160,14 @@ impl Exclusions { if self.is_empty() { return Ok(0); } - let limits = checkout.volume_config().limits; + let config = checkout.volume_config(); + let limits = config.limits; let cancel = CancellationToken::new(); let mut removed = 0; - for prefix in &self.prefixes { - let names = logical_names(prefix, limits)?; - let path = namespace(names.clone(), limits)?; + for path in &self.prefixes { + let names = path.components().to_vec(); let lookup = checkout - .lookup_no_follow(&path, WorkCounters::UNBOUNDED, &cancel) + .lookup_no_follow(path, WorkCounters::UNBOUNDED, &cancel) .await .map_err(EngineError::fs("exclusion lookup"))? .value; @@ -207,7 +178,7 @@ impl Exclusions { remove_subtree(checkout, names, limits, &cancel).await?; } checkout - .remove(path, None, WorkCounters::UNBOUNDED, &cancel) + .remove(path.clone(), None, WorkCounters::UNBOUNDED, &cancel) .await .map_err(EngineError::fs("exclusion remove"))?; removed += 1; @@ -264,41 +235,11 @@ fn remove_subtree<'a>( }) } -fn logical_names(components: &[Vec], limits: VolumeLimits) -> Result> { - components - .iter() - .map(|bytes| { - LogicalName::new( - crate::names::encoding(), - bytes.clone(), - limits.maximum_component_bytes, - ) - .map_err(|error| EngineError::Config(format!("exclude rule component: {error:?}"))) - }) - .collect() -} - fn namespace(names: Vec, limits: VolumeLimits) -> Result { NamespacePath::new(names, limits) .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}"))) } -fn host_path(components: &[Vec]) -> PathBuf { - let mut path = PathBuf::new(); - for component in components { - path.push(bytes_to_os(component)); - } - path -} - -fn os_to_bytes(name: &std::ffi::OsStr) -> Vec { - crate::names::os_to_bytes(name) -} - -fn bytes_to_os(bytes: &[u8]) -> OsString { - crate::names::bytes_to_os(bytes) -} - #[cfg(test)] mod tests { use super::*; @@ -314,12 +255,13 @@ mod tests { .split('/') .filter(|part| !part.is_empty()) .map(|part| { - LogicalName::new( - crate::names::encoding(), - crate::names::str_to_bytes(part), - limits.maximum_component_bytes, + let namespace = acyclic_fs::host_path_to_namespace( + Path::new(part), + crate::store::host_profile(), + limits, ) - .expect("name") + .expect("name"); + namespace.components()[0].clone() }) .collect(); NamespacePath::new(names, limits).expect("path") diff --git a/crates/acyclic/src/guard.rs b/crates/acyclic/src/guard.rs index c2c95c7..41c4152 100644 --- a/crates/acyclic/src/guard.rs +++ b/crates/acyclic/src/guard.rs @@ -44,18 +44,25 @@ impl GuardedPrefix { /// Splits a configured guarded path into the component bytes a /// [`MountPath`] carries. /// -/// The encoding has to be the host's (see [`crate::names`]), not UTF-8: these +/// The encoding has to be the host's, not UTF-8: these /// components are compared byte-for-byte against the names the mount layer /// hands us. Comparing UTF-8 against a host that speaks UTF-16LE never /// matches, and a guard that never matches fails *open* — every guarded path /// would silently accept writes. fn parse_guarded_path(path: &str) -> Vec> { - path.trim() - .trim_matches('/') - .split('/') - .filter(|component| !component.is_empty()) - .map(crate::names::str_to_bytes) - .collect() + let config = crate::store::volume_config(); + acyclic_fs::host_path_to_namespace( + Path::new(path.trim().trim_matches('/')), + config.profile, + config.limits, + ) + .map(|path| { + path.components() + .iter() + .map(|name| name.as_bytes().to_vec()) + .collect() + }) + .unwrap_or_default() } /// Wraps one [`MountFilesystem`] and rejects mutating calls under any @@ -124,7 +131,7 @@ impl GuardedMountFilesystem { /// bytes the mount layer carries, and on a UTF-16LE host an ASCII literal /// matches nothing. fn is_appledouble(path: &MountPath) -> bool { - let prefix = crate::names::str_to_bytes("._"); + let prefix = parse_guarded_path("._").pop().unwrap_or_default(); path.components() .last() .is_some_and(|leaf| leaf.starts_with(&prefix)) @@ -493,7 +500,8 @@ mod tests { fn test_path(components: &[&str]) -> MountPath { let mut path = MountPath::root(); for component in components { - path = path.child(crate::names::str_to_bytes(component)); + let bytes = parse_guarded_path(component).pop().expect("component"); + path = path.child(bytes); } path } diff --git a/crates/acyclic/src/lib.rs b/crates/acyclic/src/lib.rs index 69f7e97..f5d45ba 100644 --- a/crates/acyclic/src/lib.rs +++ b/crates/acyclic/src/lib.rs @@ -48,7 +48,6 @@ pub mod fork; pub mod guard; pub mod index; pub mod merge; -pub mod names; pub mod pipeline; pub mod product; pub mod rewind; diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index b0788cb..e03de29 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -25,7 +25,7 @@ use acyclic_fs::{ByteRange, CancellationToken, GenerationId, WorkCounters}; use bytes::Bytes; use crate::diff; -use crate::rewind::{namespace_path, validate_relative}; +use crate::rewind::validate_relative; use crate::store::{LocalCheckout, Store}; use crate::{EngineError, Result}; @@ -523,8 +523,10 @@ fn merged_mode(base: Option, ours: Option, theirs: Option) -> Opt } pub(crate) fn namespace_of(path: &Path) -> Result { - let components = validate_relative(path)?; - namespace_path(&components, acyclic_fs::model::VolumeLimits::default()) + validate_relative(path)?; + let config = crate::store::volume_config(); + acyclic_fs::host_path_to_namespace(path, config.profile, config.limits) + .map_err(EngineError::fs("host path to namespace")) } /// Whole content of a regular file at `path` in `checkout`. diff --git a/crates/acyclic/src/names.rs b/crates/acyclic/src/names.rs deleted file mode 100644 index 8467f9f..0000000 --- a/crates/acyclic/src/names.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! Host names ↔ engine names, in this platform's volume profile encoding. -//! -//! [`crate::store::volume_config`] picks a [`FilesystemProfile`] per platform, -//! and every layer that names a file has to agree with it: capture and -//! restore, the exclusion rules, the diff walk, and the fork mount router. -//! The mount router is the unforgiving one — the `ProjFS` provider decodes -//! every entry name it is handed as UTF-16LE, so a name that reaches it in -//! any other encoding is *projected as mojibake* rather than rejected. -//! -//! The byte form of a name is therefore platform-defined, and this module is -//! the only place that defines it: raw bytes on Unix, UTF-16LE on Windows. -//! Everywhere else in the engine a name is opaque bytes, compared and stored -//! but never interpreted. Anything that mints name bytes from host paths or -//! from configured text must come through here, or it will disagree with the -//! volume on the first non-ASCII name — and on Windows, on every name. - -use std::ffi::{OsStr, OsString}; - -use acyclic_fs::kernel::NameEncoding; -use acyclic_fs::model::FilesystemProfile; - -/// The volume profile for this host. -#[must_use] -pub const fn profile() -> FilesystemProfile { - #[cfg(windows)] - { - FilesystemProfile::Windows - } - #[cfg(unix)] - { - FilesystemProfile::Posix - } - #[cfg(not(any(unix, windows)))] - { - FilesystemProfile::Portable - } -} - -/// The name encoding that matches [`profile`]. -#[must_use] -pub const fn encoding() -> NameEncoding { - #[cfg(windows)] - { - NameEncoding::WindowsUtf16Le - } - #[cfg(unix)] - { - NameEncoding::PosixBytes - } - #[cfg(not(any(unix, windows)))] - { - NameEncoding::Utf8 - } -} - -/// Upper bound on one name component, in the bytes [`encoding`] produces. -/// -/// Both families cap a component at 255 *characters*; UTF-16LE spends two -/// bytes per unit, so the byte budget has to double on Windows or a legal -/// 200-character filename is refused as too long. -#[must_use] -pub const fn maximum_component_bytes() -> u32 { - #[cfg(windows)] - { - 510 - } - #[cfg(not(windows))] - { - 255 - } -} - -/// A host name in this platform's engine byte form. -/// -/// Lossless in both directions: Unix names are arbitrary bytes and Windows -/// names are arbitrary UTF-16, and neither is forced through UTF-8 on the -/// way past. [`bytes_to_os`] is the exact inverse. -#[cfg(unix)] -#[must_use] -pub fn os_to_bytes(name: &OsStr) -> Vec { - use std::os::unix::ffi::OsStrExt; - name.as_bytes().to_vec() -} - -#[cfg(windows)] -#[must_use] -pub fn os_to_bytes(name: &OsStr) -> Vec { - use std::os::windows::ffi::OsStrExt; - name.encode_wide().flat_map(u16::to_le_bytes).collect() -} - -#[cfg(not(any(unix, windows)))] -#[must_use] -pub fn os_to_bytes(name: &OsStr) -> Vec { - name.to_string_lossy().into_owned().into_bytes() -} - -/// Inverse of [`os_to_bytes`]. -#[cfg(unix)] -#[must_use] -pub fn bytes_to_os(bytes: &[u8]) -> OsString { - use std::os::unix::ffi::OsStringExt; - OsString::from_vec(bytes.to_vec()) -} - -/// Inverse of [`os_to_bytes`]. A trailing odd byte cannot occur in a name -/// this module produced, and is dropped rather than failing the whole walk. -#[cfg(windows)] -#[must_use] -pub fn bytes_to_os(bytes: &[u8]) -> OsString { - use std::os::windows::ffi::OsStringExt; - let (pairs, _odd_trailing_byte) = bytes.as_chunks::<2>(); - let units: Vec = pairs.iter().copied().map(u16::from_le_bytes).collect(); - OsString::from_wide(&units) -} - -#[cfg(not(any(unix, windows)))] -#[must_use] -pub fn bytes_to_os(bytes: &[u8]) -> OsString { - String::from_utf8_lossy(bytes).into_owned().into() -} - -/// UTF-8 text as engine name bytes: configured exclude rules and the fork -/// route ids the mount router projects as directory names. -#[must_use] -pub fn str_to_bytes(text: &str) -> Vec { - os_to_bytes(OsStr::new(text)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn os_bytes_round_trip_through_the_host_encoding() { - for name in ["a.txt", "sub", "ünïcøde", "日本語", ".git"] { - let bytes = os_to_bytes(OsStr::new(name)); - assert_eq!(bytes_to_os(&bytes), OsString::from(name), "{name}"); - } - } - - #[test] - fn str_bytes_agree_with_os_bytes() { - assert_eq!(str_to_bytes("fork-id"), os_to_bytes(OsStr::new("fork-id"))); - } - - /// The encoding and the profile are one decision; a host that reported - /// mismatched halves would capture names the mount layer cannot project. - #[test] - fn encoding_matches_profile() { - let expected = match profile() { - FilesystemProfile::Posix => NameEncoding::PosixBytes, - FilesystemProfile::Windows => NameEncoding::WindowsUtf16Le, - FilesystemProfile::Portable | FilesystemProfile::Browser => NameEncoding::Utf8, - }; - assert_eq!(encoding(), expected); - } - - /// An ASCII name costs one byte per character on Unix and two on - /// Windows; the budget has to cover 255 characters either way. - #[test] - fn component_budget_covers_a_maximal_host_name() { - let longest = "x".repeat(255); - let bytes = os_to_bytes(OsStr::new(&longest)); - assert!( - u32::try_from(bytes.len()).is_ok_and(|len| len <= maximum_component_bytes()), - "255-character name needs {} bytes, budget is {}", - bytes.len(), - maximum_component_bytes() - ); - } -} diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 33b3148..727f5b6 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -2218,8 +2218,9 @@ impl Pipeline { #[cfg(test)] mod root_hint_tests { use super::*; - use acyclic_fs::kernel::{LogicalName, NamespacePath}; + use acyclic_fs::kernel::NamespacePath; use acyclic_fs::{WatchEpoch, WatchSequence}; + use std::path::Path; fn root() -> NamespacePath { NamespacePath::new(Vec::new(), VolumeLimits::default()).unwrap() @@ -2227,13 +2228,8 @@ mod root_hint_tests { fn file(name: &str) -> NamespacePath { let limits = VolumeLimits::default(); - let name = LogicalName::new( - crate::names::encoding(), - crate::names::str_to_bytes(name), - limits.maximum_component_bytes, - ) - .unwrap(); - NamespacePath::new(vec![name], limits).unwrap() + acyclic_fs::host_path_to_namespace(Path::new(name), crate::store::host_profile(), limits) + .unwrap() } fn batch(changes: Vec) -> WatchBatch { diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 3b6eb5d..b0d349c 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -6,7 +6,6 @@ use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use acyclic_fs::kernel::{LogicalName, NamespacePath}; use acyclic_fs::{ exchange_native_entries, materialize_checkout, materialize_checkout_host_path, publish_native_exchange, MaterializeError, MaterializeOptions, @@ -246,26 +245,19 @@ fn create_restore_stage(parent: &Path) -> Result { } pub(crate) fn validate_relative(relative: &Path) -> Result>> { - let mut components = Vec::new(); - for component in relative.components() { - match component { - Component::Normal(name) => components.push(os_to_bytes(name)), - Component::CurDir => {} - _ => { - return Err(EngineError::Restore(format!( - "{}: path must be relative to the repo root and stay inside it", - relative.display() - ))) - } - } - } - if components.is_empty() { - return Err(EngineError::Restore(format!( - "restoring the whole tree is `{} rewind`, not a path restore", - crate::product::NAME - ))); - } - Ok(components) + let config = crate::store::volume_config(); + let namespace = acyclic_fs::host_path_to_namespace(relative, config.profile, config.limits) + .map_err(|_| { + EngineError::Restore(format!( + "{}: path must be relative to the repo root and stay inside it", + relative.display() + )) + })?; + Ok(namespace + .components() + .iter() + .map(|name| name.as_bytes().to_vec()) + .collect()) } /// Create missing ancestors one component at a time, refusing symlinks and @@ -310,25 +302,6 @@ fn ensure_real_parents(root: &Path, relative: &Path) -> Result<()> { Ok(()) } -pub(crate) fn namespace_path( - components: &[Vec], - limits: acyclic_fs::model::VolumeLimits, -) -> Result { - let names = components - .iter() - .map(|bytes| { - LogicalName::new( - crate::names::encoding(), - bytes.clone(), - limits.maximum_component_bytes, - ) - .map_err(|error| EngineError::Restore(format!("bad path component: {error:?}"))) - }) - .collect::>>()?; - NamespacePath::new(names, limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}"))) -} - pub(crate) fn remove_any(path: &Path) -> std::io::Result<()> { match std::fs::symlink_metadata(path) { Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path), @@ -338,10 +311,6 @@ pub(crate) fn remove_any(path: &Path) -> std::io::Result<()> { } } -fn os_to_bytes(name: &std::ffi::OsStr) -> Vec { - crate::names::os_to_bytes(name) -} - /// Crash-recovery journal. Present on disk only while a swap is in flight. #[derive(Debug, Serialize, Deserialize)] pub struct Journal { diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index db77514..cfa7cf7 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -1047,9 +1047,10 @@ impl Server { ) })?; let mut mount = self.fork_mount.lock().await; + let route = route_name(id)?; mount .router - .add_route(route_name(id), source) + .add_route(route.clone(), source) .map_err(|error| format!("route: {error:?}"))?; // The ONE session, mounted lazily on the first fork. A route // insert is all later forks pay. @@ -1073,7 +1074,7 @@ impl Server { match session { Ok(session) => mount.session = Some(session), Err(error) => { - tokio::task::block_in_place(|| mount.router.remove_route(&route_name(id))); + tokio::task::block_in_place(|| mount.router.remove_route(&route)); return Err(error); } } @@ -1438,7 +1439,7 @@ impl Server { /// disappears) when the last route goes, freeing the FUSE-T pool slot. async fn detach_route(&self, id: &str) -> Result<(), String> { let mut mount = self.fork_mount.lock().await; - let route = route_name(id); + let route = route_name(id)?; let last_route = mount.router.route_count() == 1; if last_route { if let Some(session) = mount.session.as_mut() { @@ -1876,8 +1877,14 @@ fn err(message: String) -> proto::Payload { /// the `ProjFS` provider decodes every entry name it is handed as UTF-16LE, /// and hands an id passed as raw ASCII back to the user as mojibake. Ids are /// hex, so this is a widening on Windows and a copy everywhere else. -fn route_name(id: &str) -> Vec { - acyclic::names::str_to_bytes(id) +fn route_name(id: &str) -> Result, String> { + let config = acyclic::store::volume_config(); + acyclic_fs::host_path_to_namespace(Path::new(id), config.profile, config.limits) + .map_err(|error| format!("route name: {error}"))? + .components() + .first() + .map(|name| name.as_bytes().to_vec()) + .ok_or_else(|| "route name is empty".to_owned()) } fn short_id() -> String { diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index 05ae9d4..3e280f0 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -190,21 +190,29 @@ pub struct Store { /// The volume every store opens with. /// /// The profile is per-platform and decides how names are encoded on the way -/// in and out — see [`crate::names`], which every caller that mints a name -/// must go through. It is fixed for the life of a volume: a store created +/// in and out. It is fixed for the life of a volume: a store created /// under one profile cannot be reopened under another, so this must never /// become a runtime choice. -pub(crate) fn volume_config() -> VolumeConfig { +pub fn volume_config() -> VolumeConfig { let mut config = VolumeConfig { - profile: crate::names::profile(), + profile: host_profile(), ..VolumeConfig::portable(Lifecycle::Durable) }; config.limits.maximum_mutations_per_batch = MUTATIONS_PER_BATCH; config.limits.maximum_paths_per_batch = MUTATIONS_PER_BATCH; - config.limits.maximum_component_bytes = crate::names::maximum_component_bytes(); + config.limits.maximum_component_bytes = if cfg!(windows) { 510 } else { 255 }; config } +pub(crate) const fn host_profile() -> acyclic_fs::model::FilesystemProfile { + #[cfg(windows)] + return acyclic_fs::model::FilesystemProfile::Windows; + #[cfg(unix)] + return acyclic_fs::model::FilesystemProfile::Posix; + #[cfg(not(any(unix, windows)))] + acyclic_fs::model::FilesystemProfile::Portable +} + pub(crate) fn writable_head() -> CheckoutMode { CheckoutMode { access: AccessMode::ReadWrite, diff --git a/crates/acyclic/tests/fork.rs b/crates/acyclic/tests/fork.rs index 8c4e3cf..e060219 100644 --- a/crates/acyclic/tests/fork.rs +++ b/crates/acyclic/tests/fork.rs @@ -19,8 +19,7 @@ use acyclic::fork::PromoteOutcome; use acyclic::index::{Attribution, CheckpointKind, Index}; use acyclic::pipeline::{self, PipelineHandle, State}; use acyclic::store::{Store, StorePaths}; -use acyclic_fs::kernel::{LogicalName, NamespacePath}; -use acyclic_fs::model::VolumeLimits; +use acyclic_fs::kernel::NamespacePath; use acyclic_fs::{CancellationToken, WorkCounters}; fn fast_config() -> Config { @@ -85,21 +84,13 @@ impl Rig { /// Built in the host's encoding rather than as a portable path: these names /// stand in for what the mount layer writes, and a diff decodes them with /// the same encoding on the way back out. -fn namespace(path: &str) -> NamespacePath { - let limits = VolumeLimits::default(); - let names = path - .split('/') - .filter(|part| !part.is_empty()) - .map(|part| { - LogicalName::new( - acyclic::names::encoding(), - acyclic::names::str_to_bytes(part), - limits.maximum_component_bytes, - ) - .expect("name") - }) - .collect(); - NamespacePath::new(names, limits).expect("namespace path") +fn namespace(path: &str, config: acyclic_fs::model::VolumeConfig) -> NamespacePath { + acyclic_fs::host_path_to_namespace( + Path::new(path.trim_start_matches('/')), + config.profile, + config.limits, + ) + .expect("namespace path") } /// Writes into a fork's overlay exactly as a mount callback would: through @@ -107,9 +98,10 @@ fn namespace(path: &str) -> NamespacePath { async fn write_in_fork(seed: &acyclic::fork::ForkSeed, path: &str, bytes: &[u8]) { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard .create_file( - namespace(path), + namespace(path, config), bytes::Bytes::copy_from_slice(bytes), WorkCounters::UNBOUNDED, &cancel, diff --git a/crates/acyclic/tests/merge.rs b/crates/acyclic/tests/merge.rs index 0e4a07c..febc61d 100644 --- a/crates/acyclic/tests/merge.rs +++ b/crates/acyclic/tests/merge.rs @@ -22,8 +22,7 @@ use acyclic::merge::{self, ConflictKind, Entry, Reason}; use acyclic::pipeline::{self, PipelineHandle}; use acyclic::store::{Store, StorePaths}; use acyclic::GenerationId; -use acyclic_fs::kernel::{LogicalName, NamespacePath}; -use acyclic_fs::model::VolumeLimits; +use acyclic_fs::kernel::NamespacePath; use acyclic_fs::{CancellationToken, WorkCounters}; fn fast_config() -> Config { @@ -117,27 +116,19 @@ impl Rig { /// Native capture stores names as POSIX bytes; lookups must use the same /// encoding or they miss (a portable-encoded name is a different key). -fn namespace(path: &str) -> NamespacePath { - let limits = VolumeLimits::default(); - let names = path - .trim_start_matches('/') - .split('/') - .map(|component| { - LogicalName::new( - acyclic::names::encoding(), - acyclic::names::str_to_bytes(component), - limits.maximum_component_bytes, - ) - .expect("logical name") - }) - .collect(); - NamespacePath::new(names, limits).expect("namespace path") +fn namespace(path: &str, config: acyclic_fs::model::VolumeConfig) -> NamespacePath { + acyclic_fs::host_path_to_namespace( + Path::new(path.trim_start_matches('/')), + config.profile, + config.limits, + ) + .expect("namespace path") } async fn fork_write(seed: &ForkSeed, path: &str, bytes: &[u8]) { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; - let ns = namespace(path); + let ns = namespace(path, guard.volume_config()); let existing = guard .lookup_no_follow(&ns, WorkCounters::UNBOUNDED, &cancel) .await @@ -163,8 +154,14 @@ async fn fork_write(seed: &ForkSeed, path: &str, bytes: &[u8]) { async fn fork_remove(seed: &ForkSeed, path: &str) { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard - .remove(namespace(path), None, WorkCounters::UNBOUNDED, &cancel) + .remove( + namespace(path, config), + None, + WorkCounters::UNBOUNDED, + &cancel, + ) .await .expect("remove in fork overlay"); } @@ -460,9 +457,10 @@ fn refusals_name_paths_and_reasons() { { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard .remove( - namespace("/src/same.txt"), + namespace("/src/same.txt", config), None, WorkCounters::UNBOUNDED, &cancel, @@ -471,7 +469,7 @@ fn refusals_name_paths_and_reasons() { .expect("rm"); guard .create_symbolic_link( - namespace("/src/same.txt"), + namespace("/src/same.txt", config), bytes::Bytes::from_static(b"shared.txt"), WorkCounters::UNBOUNDED, &cancel, @@ -578,16 +576,25 @@ fn nested_directory_subtree_copies_into_merge_generation() { { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard - .create_directory(namespace("/docs"), WorkCounters::UNBOUNDED, &cancel) + .create_directory(namespace("/docs", config), WorkCounters::UNBOUNDED, &cancel) .await .expect("mkdir"); guard - .create_directory(namespace("/docs/deep"), WorkCounters::UNBOUNDED, &cancel) + .create_directory( + namespace("/docs/deep", config), + WorkCounters::UNBOUNDED, + &cancel, + ) .await .expect("mkdir"); guard - .create_directory(namespace("/docs/deep/er"), WorkCounters::UNBOUNDED, &cancel) + .create_directory( + namespace("/docs/deep/er", config), + WorkCounters::UNBOUNDED, + &cancel, + ) .await .expect("mkdir"); } @@ -634,12 +641,17 @@ fn materialize_paths_writes_a_generation_into_a_directory() { { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard - .create_directory(namespace("/docs"), WorkCounters::UNBOUNDED, &cancel) + .create_directory(namespace("/docs", config), WorkCounters::UNBOUNDED, &cancel) .await .expect("mkdir"); guard - .create_directory(namespace("/docs/deep"), WorkCounters::UNBOUNDED, &cancel) + .create_directory( + namespace("/docs/deep", config), + WorkCounters::UNBOUNDED, + &cancel, + ) .await .expect("mkdir"); } From 1f6947838564f127f9ea3ccd29fb0926e6d42427 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 11:49:26 +0100 Subject: [PATCH 061/106] Capture restored subtrees without watcher delay --- crates/acyclic/src/pipeline.rs | 35 +++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 727f5b6..a257386 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -9,7 +9,9 @@ use std::path::PathBuf; use std::time::{Duration, Instant}; use acyclic_fs::model::VolumeLimits; -use acyclic_fs::{capture_baseline, capture_root_identity, capture_watch_batch, CaptureOptions}; +use acyclic_fs::{ + capture_baseline, capture_root_identity, capture_subtree, capture_watch_batch, CaptureOptions, +}; use acyclic_fs::{ CancellationToken, CheckoutCommitOutcome, GenerationId, MountPublication, NativeWatch, NativeWatchOptions, OperationId, WatchBatch, WatchChange, WatchEpoch, WatchSequence, @@ -1793,6 +1795,26 @@ impl Pipeline { self.scrub_exclusions().await } + /// Reconciles exact restored roots immediately, including all descendants + /// of a directory that was replaced or removed. The SDK unions the host + /// and checkout subtrees, so stale descendants are removed without waiting + /// for the native watcher to enumerate the write. + async fn capture_restored_subtrees(&mut self, paths: &[PathBuf]) -> Result<()> { + for path in paths { + let namespace = crate::merge::namespace_of(path)?; + capture_subtree( + &mut self.store.checkout, + namespace, + &self.options, + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await + .map_err(EngineError::fs("capture restored subtree"))?; + } + self.scrub_exclusions().await + } + /// Restores `paths` from `target` as one event: at most one safety row /// before, one drain and one row after, however many paths land. async fn restore_paths( @@ -1853,10 +1875,9 @@ impl Pipeline { // later. The echo is harmless when it comes: a modified hint on a // path whose content already matches captures nothing, and the // staged sibling's create+rename resolves to an absent path. - // Direct capture describes each path with one hint, which is exact - // for a regular file or symlink and wrong for a subtree (a removed - // or replaced directory needs a hint per descendant). Anything - // else waits for the native watcher, which delivers those. + // Leaf batches avoid directory traversal. Directory and absent roots + // use the SDK subtree reconciler, which unions live and stored + // descendants and therefore captures replacements and removals exactly. let post_started = Instant::now(); let all_leaves = paths.iter().all(|path| { std::fs::symlink_metadata(self.store.repo_root.join(path)) @@ -1866,8 +1887,8 @@ impl Pipeline { self.capture_paths_directly(paths).await?; "direct capture" } else { - self.drain_watcher().await?; - "watcher drain (a path is a directory or absent)" + self.capture_restored_subtrees(paths).await?; + "direct subtree capture" }; let post_ms = crate::trace::ms(post_started); let capture_started = Instant::now(); From de04c943895b334151444d19852e90eb990ba915 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 11:52:42 +0100 Subject: [PATCH 062/106] Deduplicate overlapping restore roots --- crates/acyclic/src/pipeline.rs | 11 ++++++----- crates/acyclic/tests/pipeline.rs | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index a257386..050a410 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -1824,7 +1824,8 @@ impl Pipeline { safety: bool, label: Option, ) -> Result { - for path in paths { + let paths = crate::merge::subtree_roots(paths); + for path in &paths { if self.exclusions.covers_host(path) { return Err(EngineError::Restore(format!( "{} is excluded from snapshots (`exclude` in {}); no checkpoint holds it", @@ -1837,7 +1838,7 @@ impl Pipeline { self.reset_watch().await?; self.baseline(CheckpointKind::Recovered).await?; } - let what = match paths { + let what = match paths.as_slice() { [path] => format!("{} from #{}", path.display(), target.id), _ => format!("{} path(s) from #{}", paths.len(), target.id), }; @@ -1865,7 +1866,7 @@ impl Pipeline { let write_started = Instant::now(); let mut outcomes = Vec::with_capacity(paths.len()); - for path in paths { + for path in &paths { outcomes.push(rewind::restore_path(&self.store, target.generation, path).await?); } let write_ms = crate::trace::ms(write_started); @@ -1884,10 +1885,10 @@ impl Pipeline { .is_ok_and(|metadata| metadata.is_file() || metadata.is_symlink()) }); let capture_mode = if all_leaves { - self.capture_paths_directly(paths).await?; + self.capture_paths_directly(&paths).await?; "direct capture" } else { - self.capture_restored_subtrees(paths).await?; + self.capture_restored_subtrees(&paths).await?; "direct subtree capture" }; let post_ms = crate::trace::ms(post_started); diff --git a/crates/acyclic/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs index ac944d6..100bf60 100644 --- a/crates/acyclic/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -604,6 +604,29 @@ fn single_path_restore_leaves_the_rest_alone() { b"changed\n" ); + // Batched callers may contain duplicates and descendants of a root. + // The pipeline must materialize and reconcile the minimal root once. + std::fs::write(root.join("src/deep/a.txt"), b"a3\n").expect("edit again"); + let restored = handle + .restore_paths( + target(v1.row_id), + vec![ + "src/deep/a.txt".into(), + "src/deep".into(), + "src/deep".into(), + ], + false, + Some("deduplicated restore".into()), + ) + .await + .expect("restore minimal roots"); + assert_eq!(restored.outcomes.len(), 1); + assert_eq!(restored.outcomes[0].path, Path::new("src/deep")); + assert_eq!( + std::fs::read(root.join("src/deep/a.txt")).expect("read"), + b"a1\n" + ); + // A path absent at the checkpoint is removed. let outcome = handle .restore_path(target(v1.row_id), "new.txt".into()) From 98b06bcc80eb509aeaa06217478be4ec1302ea44 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:01:15 +0100 Subject: [PATCH 063/106] Linearize restore root reduction --- crates/acyclic/src/merge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index e03de29..65e1fc6 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -1007,7 +1007,7 @@ pub fn subtree_roots(paths: &[PathBuf]) -> Vec { sorted.sort(); let mut roots: Vec = Vec::new(); for path in sorted { - if !roots.iter().any(|root| path.starts_with(root)) { + if roots.last().is_none_or(|root| !path.starts_with(root)) { roots.push(path); } } From f56d707556cfb3f39ce876c4415096991ae70e7e Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:10:39 +0100 Subject: [PATCH 064/106] Centralize capture exclusions in SDK --- crates/acyclic/src/exclude.rs | 275 ++++----------------------------- crates/acyclic/src/pipeline.rs | 48 +++--- 2 files changed, 49 insertions(+), 274 deletions(-) diff --git a/crates/acyclic/src/exclude.rs b/crates/acyclic/src/exclude.rs index 636c46f..77c1323 100644 --- a/crates/acyclic/src/exclude.rs +++ b/crates/acyclic/src/exclude.rs @@ -20,15 +20,11 @@ use std::path::{Path, PathBuf}; -use acyclic_fs::kernel::{FileKind, LogicalName, NamespacePath}; -use acyclic_fs::model::VolumeLimits; -use acyclic_fs::{CancellationToken, WatchBatch, WatchChange, WorkCounters}; +use acyclic_fs::kernel::NamespacePath; +use acyclic_fs::CapturePolicy; -use crate::store::LocalCheckout; use crate::{EngineError, Result}; -const PAGE_ENTRIES: u32 = 1_024; - /// Parsed `exclude` rules: repo-relative path prefixes. A rule matches the /// path itself and everything under it; `secrets` and `secrets/` are the /// same rule. @@ -56,11 +52,22 @@ impl Exclusions { "exclude rule {pattern:?} must be a non-empty relative path inside the repo" )) })?; - if !prefixes.contains(&prefix) { - prefixes.push(prefix); + prefixes.push(prefix); + } + prefixes.sort(); + prefixes.dedup(); + let mut canonical = Vec::::new(); + for prefix in prefixes { + if canonical + .last() + .is_none_or(|ancestor| !prefix.is_within(ancestor)) + { + canonical.push(prefix); } } - Ok(Self { prefixes }) + Ok(Self { + prefixes: canonical, + }) } pub fn is_empty(&self) -> bool { @@ -84,205 +91,30 @@ impl Exclusions { /// True when `path` is an excluded path or lies under one. pub fn covers(&self, path: &NamespacePath) -> bool { - self.prefixes.iter().any(|prefix| path.is_within(prefix)) - } - - /// True when `path` is a strict ancestor of an excluded path (the repo - /// root included). A capture hinted at such a path may re-walk the - /// excluded subtree, so the checkout needs a scrub afterwards. - fn is_ancestor(&self, path: &NamespacePath) -> bool { - self.prefixes - .iter() - .any(|prefix| path.depth() < prefix.depth() && prefix.is_within(path)) - } - - /// Drops hints the capture must not read and rewrites renames that - /// cross the exclusion boundary so the uncovered side is re-examined. - /// Returns the batch and whether a scrub is needed after capturing it. - pub fn filter_batch(&self, batch: WatchBatch) -> (WatchBatch, bool) { - if self.is_empty() { - return (batch, false); - } - let WatchBatch::Changes { - epoch, - first_sequence, - next_sequence, - changes, - } = batch - else { - return (batch, false); - }; - let mut scrub = false; - let mut kept = Vec::with_capacity(changes.len()); - for change in changes { - match change { - WatchChange::Created(path) - | WatchChange::Modified(path) - | WatchChange::MetadataChanged(path) - | WatchChange::Removed(path) => { - if self.covers(&path) { - continue; - } - scrub |= self.is_ancestor(&path); - kept.push(WatchChange::Modified(path)); - } - WatchChange::Renamed { from, to } => match (self.covers(&from), self.covers(&to)) { - (true, true) => {} - (true, false) => { - scrub |= self.is_ancestor(&to); - kept.push(WatchChange::Modified(to)); - } - (false, true) => { - scrub |= self.is_ancestor(&from); - kept.push(WatchChange::Modified(from)); - } - (false, false) => { - scrub |= self.is_ancestor(&from) || self.is_ancestor(&to); - kept.push(WatchChange::Renamed { from, to }); - } - }, - } - } - ( - WatchBatch::Changes { - epoch, - first_sequence, - next_sequence, - changes: kept, - }, - scrub, - ) + let candidate = self + .prefixes + .partition_point(|prefix| prefix <= path) + .checked_sub(1) + .and_then(|index| self.prefixes.get(index)); + candidate.is_some_and(|prefix| path.is_within(prefix)) } - /// Removes every excluded path that is present in the checkout. Returns - /// how many rules had something to remove. - pub async fn scrub(&self, checkout: &mut LocalCheckout) -> Result { - if self.is_empty() { - return Ok(0); - } - let config = checkout.volume_config(); - let limits = config.limits; - let cancel = CancellationToken::new(); - let mut removed = 0; - for path in &self.prefixes { - let names = path.components().to_vec(); - let lookup = checkout - .lookup_no_follow(path, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("exclusion lookup"))? - .value; - let Some(record) = lookup.record else { - continue; - }; - if record.kind == FileKind::Directory { - remove_subtree(checkout, names, limits, &cancel).await?; - } - checkout - .remove(path.clone(), None, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("exclusion remove"))?; - removed += 1; - } - Ok(removed) + /// Canonical SDK capture policy shared by baseline, watcher, and direct + /// subtree reconciliation. + pub fn capture_policy(&self) -> Result { + CapturePolicy::excluding(self.prefixes.clone()) + .map_err(|error| EngineError::Fs(format!("capture policy: {error}"))) } } -/// Post-order removal of a directory's contents (the fs refuses to unbind a -/// non-empty directory). The directory binding itself is left to the caller. -fn remove_subtree<'a>( - checkout: &'a mut LocalCheckout, - directory: Vec, - limits: VolumeLimits, - cancel: &'a CancellationToken, -) -> std::pin::Pin> + 'a>> { - Box::pin(async move { - let path = namespace(directory.clone(), limits)?; - let mut entries = Vec::new(); - let mut after = None; - loop { - let page = checkout - .list_directory_records( - &path, - after.as_ref(), - PAGE_ENTRIES, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("exclusion list"))? - .value; - for entry in &page.entries { - entries.push((entry.name.clone(), entry.record.kind)); - } - match page.entries.last() { - Some(last) if page.has_more => after = Some(last.name.clone()), - _ => break, - } - } - for (name, kind) in entries { - let mut child = directory.clone(); - child.push(name); - if kind == FileKind::Directory { - remove_subtree(checkout, child.clone(), limits, cancel).await?; - } - let child_path = namespace(child, limits)?; - checkout - .remove(child_path, None, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("exclusion remove"))?; - } - Ok(()) - }) -} - -fn namespace(names: Vec, limits: VolumeLimits) -> Result { - NamespacePath::new(names, limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}"))) -} - #[cfg(test)] mod tests { use super::*; - use acyclic_fs::{WatchEpoch, WatchSequence}; fn rules(list: &[&str]) -> Exclusions { Exclusions::parse(&list.iter().map(|s| s.to_string()).collect::>()).expect("parse") } - fn ns(path: &str) -> NamespacePath { - let limits = VolumeLimits::default(); - let names = path - .split('/') - .filter(|part| !part.is_empty()) - .map(|part| { - let namespace = acyclic_fs::host_path_to_namespace( - Path::new(part), - crate::store::host_profile(), - limits, - ) - .expect("name"); - namespace.components()[0].clone() - }) - .collect(); - NamespacePath::new(names, limits).expect("path") - } - - fn batch(changes: Vec) -> WatchBatch { - WatchBatch::Changes { - epoch: WatchEpoch::from_u64(1), - first_sequence: WatchSequence::from_u64(1), - next_sequence: WatchSequence::from_u64(2), - changes, - } - } - - fn changes(batch: &WatchBatch) -> &[WatchChange] { - match batch { - WatchBatch::Changes { changes, .. } => changes, - WatchBatch::RescanRequired { .. } => panic!("rescan"), - } - } - #[test] fn rules_normalize_and_reject_escapes() { let parsed = rules(&[".env", "secrets/", "./build/out/"]); @@ -299,58 +131,13 @@ mod tests { parsed.host_paths(), vec![ PathBuf::from(".env"), - PathBuf::from("secrets"), - PathBuf::from("build/out") + PathBuf::from("build/out"), + PathBuf::from("secrets") ] ); - } - - #[test] - fn covered_hints_are_dropped_and_ancestors_demand_a_scrub() { - let parsed = rules(&["secrets"]); - let (filtered, scrub) = parsed.filter_batch(batch(vec![ - WatchChange::Created(ns("secrets/key.pem")), - WatchChange::Modified(ns("src/main.rs")), - ])); assert_eq!( - changes(&filtered), - &[WatchChange::Modified(ns("src/main.rs"))] + rules(&["secrets/deep", "secrets", "secrets/deeper"]).host_paths(), + vec![PathBuf::from("secrets")] ); - assert!(!scrub); - - let (filtered, scrub) = parsed.filter_batch(batch(vec![WatchChange::Modified(ns(""))])); - assert_eq!(changes(&filtered).len(), 1); - assert!(scrub, "a root hint may re-walk the excluded subtree"); - } - - #[test] - fn renames_across_the_boundary_reexamine_the_uncovered_side() { - let parsed = rules(&["secrets"]); - let (filtered, _) = parsed.filter_batch(batch(vec![WatchChange::Renamed { - from: ns("staging"), - to: ns("secrets"), - }])); - assert_eq!(changes(&filtered), &[WatchChange::Modified(ns("staging"))]); - - let (filtered, _) = parsed.filter_batch(batch(vec![WatchChange::Renamed { - from: ns("secrets"), - to: ns("public"), - }])); - assert_eq!(changes(&filtered), &[WatchChange::Modified(ns("public"))]); - - let (filtered, _) = parsed.filter_batch(batch(vec![WatchChange::Renamed { - from: ns("secrets/a"), - to: ns("secrets/b"), - }])); - assert!(changes(&filtered).is_empty()); - } - - #[test] - fn empty_rules_pass_batches_through_untouched() { - let parsed = rules(&[]); - let original = batch(vec![WatchChange::Created(ns("anything"))]); - let (filtered, scrub) = parsed.filter_batch(original.clone()); - assert_eq!(filtered, original); - assert!(!scrub); } } diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 050a410..c2695cc 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -10,7 +10,8 @@ use std::time::{Duration, Instant}; use acyclic_fs::model::VolumeLimits; use acyclic_fs::{ - capture_baseline, capture_root_identity, capture_subtree, capture_watch_batch, CaptureOptions, + capture_baseline_with_policy, capture_root_identity, capture_subtree_with_policy, + capture_watch_batch_with_policy, CaptureOptions, }; use acyclic_fs::{ CancellationToken, CheckoutCommitOutcome, GenerationId, MountPublication, NativeWatch, @@ -729,6 +730,7 @@ struct Pipeline { options: CaptureOptions, /// Paths that never enter a checkpoint (`exclude` in the config). exclusions: Exclusions, + capture_policy: acyclic_fs::CapturePolicy, cancel: CancellationToken, state: State, last_generation: GenerationId, @@ -848,6 +850,7 @@ impl Pipeline { }; let exclusions = Exclusions::parse(&config.exclude)?; + let capture_policy = exclusions.capture_policy()?; let pipeline = Self { store, index, @@ -855,6 +858,7 @@ impl Pipeline { watch: None, options, exclusions, + capture_policy, cancel, state: State::NeedsBaseline, last_generation: GenerationId::new(acyclic_fs::Digest::ZERO), @@ -956,18 +960,16 @@ impl Pipeline { self.commit_engine().await?; let precommit_ms = crate::trace::ms(phase); let phase = Instant::now(); - capture_baseline( + capture_baseline_with_policy( &mut self.store.checkout, &self.options, + &self.capture_policy, WorkCounters::UNBOUNDED, &self.cancel, ) .await .map_err(EngineError::fs("capture baseline"))?; let capture_ms = crate::trace::ms(phase); - let phase = Instant::now(); - self.scrub_exclusions().await?; - let scrub_ms = crate::trace::ms(phase); // Changes that raced the baseline arrive as the rescan-completion // batch; fold them in before declaring Ready. let batch = self @@ -997,10 +999,11 @@ impl Pipeline { } if let WatchBatch::Changes { ref changes, .. } = batch { if !changes.is_empty() { - let captured = capture_watch_batch( + let captured = capture_watch_batch_with_policy( &mut self.store.checkout, batch, &self.options, + &self.capture_policy, WorkCounters::UNBOUNDED, &self.cancel, ) @@ -1022,7 +1025,6 @@ impl Pipeline { } return Err(EngineError::fs("capture rescan tail")(failure)); } - self.scrub_exclusions().await?; } } let phase = Instant::now(); @@ -1058,7 +1060,7 @@ impl Pipeline { crate::trace!( "pipeline", "baseline phases: pre-commit {precommit_ms:.1}ms, full capture {capture_ms:.1}ms, \ - scrub {scrub_ms:.1}ms, snapshot {snapshot_ms:.1}ms, post-commit {postcommit_ms:.1}ms" + snapshot {snapshot_ms:.1}ms, post-commit {postcommit_ms:.1}ms" ); crate::trace!( "pipeline", @@ -1430,7 +1432,6 @@ impl Pipeline { "drain: watcher hinted at the volume root ({root:?}); dropped" ); } - let (batch, scrub) = self.exclusions.filter_batch(batch); if root == RootHint::Structural { // The repo directory itself changed identity (a mount came // or went): re-baseline on a fresh watcher rather than apply @@ -1447,10 +1448,11 @@ impl Pipeline { WatchBatch::Changes { ref changes, .. } if !changes.is_empty() => { batches += 1; hints += changes.len(); - let captured = capture_watch_batch( + let captured = capture_watch_batch_with_policy( &mut self.store.checkout, batch, &self.options, + &self.capture_policy, WorkCounters::UNBOUNDED, &self.cancel, ) @@ -1468,9 +1470,6 @@ impl Pipeline { } return Err(EngineError::fs("capture watch batch")(failure)); } - if scrub { - self.scrub_exclusions().await?; - } changed = true; last_change = Some(Instant::now()); } @@ -1518,19 +1517,6 @@ impl Pipeline { } } - /// Drops excluded paths a capture may have pulled into the checkout, - /// before the generation they would otherwise land in is checkpointed. - async fn scrub_exclusions(&mut self) -> Result<()> { - let removed = self.exclusions.scrub(&mut self.store.checkout).await?; - if removed > 0 { - crate::trace!( - "pipeline", - "exclusions: scrubbed {removed} excluded path(s) from the checkout" - ); - } - Ok(()) - } - /// `checkpoint()`: snapshot without authority publish. The fast path. async fn checkpoint_engine(&mut self) -> Result { Ok(self @@ -1766,7 +1752,7 @@ impl Pipeline { &parent, )?)); } - let captured = capture_watch_batch( + let captured = capture_watch_batch_with_policy( &mut self.store.checkout, WatchBatch::Changes { epoch: WatchEpoch::from_u64(0), @@ -1775,6 +1761,7 @@ impl Pipeline { changes: hints, }, &self.options, + &self.capture_policy, WorkCounters::UNBOUNDED, &self.cancel, ) @@ -1792,7 +1779,7 @@ impl Pipeline { } return Err(EngineError::fs("capture restored paths")(failure)); } - self.scrub_exclusions().await + Ok(()) } /// Reconciles exact restored roots immediately, including all descendants @@ -1802,17 +1789,18 @@ impl Pipeline { async fn capture_restored_subtrees(&mut self, paths: &[PathBuf]) -> Result<()> { for path in paths { let namespace = crate::merge::namespace_of(path)?; - capture_subtree( + capture_subtree_with_policy( &mut self.store.checkout, namespace, &self.options, + &self.capture_policy, WorkCounters::UNBOUNDED, &self.cancel, ) .await .map_err(EngineError::fs("capture restored subtree"))?; } - self.scrub_exclusions().await + Ok(()) } /// Restores `paths` from `target` as one event: at most one safety row From 786fd6b5485df88b53e1934f7f505a512de7da09 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:16:59 +0100 Subject: [PATCH 065/106] Batch restored subtree capture --- crates/acyclic/src/pipeline.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index c2695cc..8568899 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -10,7 +10,7 @@ use std::time::{Duration, Instant}; use acyclic_fs::model::VolumeLimits; use acyclic_fs::{ - capture_baseline_with_policy, capture_root_identity, capture_subtree_with_policy, + capture_baseline_with_policy, capture_root_identity, capture_subtrees_with_policy, capture_watch_batch_with_policy, CaptureOptions, }; use acyclic_fs::{ @@ -1787,19 +1787,20 @@ impl Pipeline { /// and checkout subtrees, so stale descendants are removed without waiting /// for the native watcher to enumerate the write. async fn capture_restored_subtrees(&mut self, paths: &[PathBuf]) -> Result<()> { - for path in paths { - let namespace = crate::merge::namespace_of(path)?; - capture_subtree_with_policy( - &mut self.store.checkout, - namespace, - &self.options, - &self.capture_policy, - WorkCounters::UNBOUNDED, - &self.cancel, - ) - .await - .map_err(EngineError::fs("capture restored subtree"))?; - } + let roots = paths + .iter() + .map(|path| crate::merge::namespace_of(path)) + .collect::>>()?; + capture_subtrees_with_policy( + &mut self.store.checkout, + &roots, + &self.options, + &self.capture_policy, + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await + .map_err(EngineError::fs("capture restored subtrees"))?; Ok(()) } From 46d7a43c9c49afc4c6da121776e0746ce17c3723 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:21:47 +0100 Subject: [PATCH 066/106] Batch generation file lookups --- crates/acyclic/src/merge.rs | 48 +++++++++++++++++++++++++--------- crates/acyclic/src/pipeline.rs | 9 +++---- crates/acyclic/tests/merge.rs | 26 ++++++++++++++++++ 3 files changed, 64 insertions(+), 19 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 65e1fc6..f924e66 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -541,16 +541,26 @@ pub(crate) async fn read_regular(checkout: &mut LocalCheckout, path: &Path) -> R let record = lookup .record .ok_or_else(|| EngineError::Fs(format!("{}: absent", path.display())))?; + read_regular_record(checkout, &namespace, record, path).await +} + +async fn read_regular_record( + checkout: &mut LocalCheckout, + namespace: &NamespacePath, + record: FileRecord, + display_path: &Path, +) -> Result> { let length = match record.payload { acyclic_fs::kernel::FilePayload::InlineRegular(inline) => inline.as_bytes().len() as u64, acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, _ => { return Err(EngineError::Fs(format!( "{}: not a regular file", - path.display() + display_path.display() ))) } }; + let cancel = CancellationToken::new(); let limits = checkout.volume_config().limits; let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); let mut out = Vec::with_capacity(usize::try_from(length).unwrap_or(0)); @@ -559,7 +569,7 @@ pub(crate) async fn read_regular(checkout: &mut LocalCheckout, path: &Path) -> R let take = chunk.min(length - offset); let read = checkout .read_file_range( - &namespace, + namespace, ByteRange { offset, length: take, @@ -576,26 +586,38 @@ pub(crate) async fn read_regular(checkout: &mut LocalCheckout, path: &Path) -> R Ok(out) } -/// Content of `path` in `generation` if it is a regular file there. -pub async fn read_file( +/// Contents of regular files in one generation, preserving request order. +/// Opens and authenticates the generation once and resolves every namespace +/// path in one SDK batch; absent and non-regular entries remain `None`. +pub async fn read_files( store: &Store, generation: GenerationId, - path: &Path, -) -> Result>> { + paths: &[PathBuf], +) -> Result>>> { let mut checkout = store.checkout_exact(generation).await?; + let namespaces = paths + .iter() + .map(|path| namespace_of(path)) + .collect::>>()?; + if namespaces.is_empty() { + return Ok(Vec::new()); + } let cancel = CancellationToken::new(); - let namespace = namespace_of(path)?; let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) + .lookup_batch_no_follow(&namespaces, WorkCounters::UNBOUNDED, &cancel) .await - .map_err(EngineError::fs("lookup"))? + .map_err(EngineError::fs("batch lookup"))? .value; - match lookup.record { - Some(record) if record.kind == FileKind::Regular => { - Ok(Some(read_regular(&mut checkout, path).await?)) + let mut contents = Vec::with_capacity(paths.len()); + for ((path, namespace), entry) in paths.iter().zip(namespaces).zip(lookup.entries) { + match entry.record { + Some(record) if record.kind == FileKind::Regular => contents.push(Some( + read_regular_record(&mut checkout, &namespace, record, path).await?, + )), + _ => contents.push(None), } - _ => Ok(None), } + Ok(contents) } async fn read_mode(checkout: &mut LocalCheckout, path: &Path) -> Result> { diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 8568899..96f764d 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -1253,12 +1253,9 @@ impl Pipeline { reply, } => { let result = Box::pin(async { - let mut out = Vec::with_capacity(paths.len()); - for path in paths { - let bytes = crate::merge::read_file(&self.store, generation, &path).await?; - out.push((path, bytes)); - } - Ok(out) + let contents = + crate::merge::read_files(&self.store, generation, &paths).await?; + Ok(paths.into_iter().zip(contents).collect()) }) .await; let _ = reply.send(result); diff --git a/crates/acyclic/tests/merge.rs b/crates/acyclic/tests/merge.rs index febc61d..415175a 100644 --- a/crates/acyclic/tests/merge.rs +++ b/crates/acyclic/tests/merge.rs @@ -186,6 +186,32 @@ fn p(s: &str) -> PathBuf { PathBuf::from(s) } +#[test] +fn batch_reads_preserve_order_and_non_regular_absence() { + let rig = Rig::start(); + rig.runtime.block_on(async { + let generation = rig.handle.publish_head().await.expect("head"); + let contents = rig + .handle + .read_files( + generation, + vec![p("src/shared.txt"), p("missing"), p("src"), p("bin.dat")], + ) + .await + .expect("batch read"); + assert_eq!( + contents, + vec![ + (p("src/shared.txt"), Some(b"1\n2\n3\n".to_vec())), + (p("missing"), None), + (p("src"), None), + (p("bin.dat"), Some(b"\x00\x01\x02".to_vec())), + ] + ); + }); + rig.finish(); +} + /// The plan over a realistic overlap: disjoint paths on both sides, a file /// merged by content, an identical change, and independent additions under /// one directory. Then M = H + entries holds exactly the expected tree. From 81f18e76c136d82cd15e459ca5146fbadaef6135 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:30:51 +0100 Subject: [PATCH 067/106] Overlap generation file reads --- crates/acyclic/src/merge.rs | 79 +++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index f924e66..f35e289 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -21,7 +21,9 @@ pub use acyclic_fs::text_merge::{ use acyclic_fs::text_merge::{ merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, }; -use acyclic_fs::{ByteRange, CancellationToken, GenerationId, WorkCounters}; +use acyclic_fs::{ + ByteRange, CancellationToken, FileRecordRangeReadRequest, GenerationId, WorkCounters, +}; use bytes::Bytes; use crate::diff; @@ -32,6 +34,7 @@ use crate::{EngineError, Result}; /// Default `[merge] max_file_bytes`. pub const DEFAULT_MAX_FILE_BYTES: u64 = 4 * 1024 * 1024; const TRANSFER_BYTES: u64 = 8 * 1024 * 1024; +const FILE_READ_CONCURRENCY: usize = 32; const PAGE_ENTRIES: u32 = 1_024; // --------------------------------------------------------------------------- // Per-path decision table @@ -608,15 +611,75 @@ pub async fn read_files( .await .map_err(EngineError::fs("batch lookup"))? .value; - let mut contents = Vec::with_capacity(paths.len()); - for ((path, namespace), entry) in paths.iter().zip(namespaces).zip(lookup.entries) { - match entry.record { - Some(record) if record.kind == FileKind::Regular => contents.push(Some( - read_regular_record(&mut checkout, &namespace, record, path).await?, - )), - _ => contents.push(None), + let limits = checkout.volume_config().limits; + let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); + let mut contents = vec![None; paths.len()]; + let mut requests = Vec::new(); + let mut destinations = Vec::new(); + for (index, entry) in lookup.entries.into_iter().enumerate() { + let Some(record) = entry + .record + .filter(|record| record.kind == FileKind::Regular) + else { + continue; + }; + let length = match record.payload { + acyclic_fs::kernel::FilePayload::InlineRegular(inline) => { + inline.as_bytes().len() as u64 + } + acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, + _ => unreachable!("regular records have regular payloads"), + }; + let display_path = paths + .get(index) + .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))?; + let capacity = usize::try_from(length).map_err(|_| { + EngineError::Fs(format!( + "{}: file is too large for this host", + display_path.display() + )) + })?; + let content = contents + .get_mut(index) + .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))?; + *content = Some(Vec::with_capacity(capacity)); + let mut offset = 0; + while offset < length { + let take = chunk.min(length - offset); + requests.push(FileRecordRangeReadRequest { + record, + range: ByteRange { + offset, + length: take, + }, + }); + destinations.push(index); + offset += take; } } + if requests.is_empty() { + return Ok(contents); + } + let reader = checkout + .pinned_reader() + .map_err(EngineError::fs("open pinned reader"))?; + let reads = reader + .read_file_record_ranges( + &requests, + FILE_READ_CONCURRENCY, + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(EngineError::fs("batch read file ranges"))? + .value; + for (destination, read) in destinations.into_iter().zip(reads) { + contents + .get_mut(destination) + .and_then(Option::as_mut) + .ok_or_else(|| EngineError::Fs("batch read returned an invalid destination".into()))? + .extend_from_slice(&read.bytes); + } Ok(contents) } From ca03a9825fb372734909202c746f176227872818 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:34:24 +0100 Subject: [PATCH 068/106] Batch merge planning content reads --- crates/acyclic/src/merge.rs | 122 ++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 69 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index f35e289..ad2cba3 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -300,6 +300,41 @@ struct MergeInputs { theirs_changed: BTreeSet, } +fn regular_paths(records: &BTreeMap) -> Vec { + records + .iter() + .filter(|(_, record)| record.kind == FileKind::Regular) + .map(|(path, _)| path.clone()) + .collect() +} + +async fn regular_contents( + store: &Store, + generation: GenerationId, + records: &BTreeMap, +) -> Result>> { + let paths = regular_paths(records); + let contents = read_files(store, generation, &paths).await?; + paths + .into_iter() + .zip(contents) + .map(|(path, content)| { + content + .map(|content| (path.clone(), content)) + .ok_or_else(|| { + EngineError::Fs(format!("{}: regular file is absent", path.display())) + }) + }) + .collect() +} + +fn regular_content<'a>(contents: &'a BTreeMap>, path: &Path) -> Result<&'a [u8]> { + contents + .get(path) + .map(Vec::as_slice) + .ok_or_else(|| EngineError::Fs(format!("{}: regular content is absent", path.display()))) +} + async fn merge_inputs( store: &Store, base: GenerationId, @@ -343,6 +378,12 @@ pub async fn plan( theirs_changed, } = inputs; + let (base_contents, theirs_contents, ours_contents) = tokio::try_join!( + regular_contents(store, base, &base_map), + regular_contents(store, theirs, &theirs_map), + regular_contents(store, ours, &ours_map), + )?; + let mut base_checkout = store.checkout_exact(base).await?; let mut theirs_checkout = store.checkout_exact(theirs).await?; let mut ours_checkout = store.checkout_exact(ours).await?; @@ -389,7 +430,7 @@ pub async fn plan( continue; } let base_bytes = match kind(b) { - Some(FileKind::Regular) => Some(read_regular(&mut base_checkout, path).await?), + Some(FileKind::Regular) => Some(regular_content(&base_contents, path)?), None => None, Some(_) => { plan.refusals.push(Refusal { @@ -399,8 +440,8 @@ pub async fn plan( continue; } }; - let ours_bytes = read_regular(&mut ours_checkout, path).await?; - let theirs_bytes = read_regular(&mut theirs_checkout, path).await?; + let ours_bytes = regular_content(&ours_contents, path)?; + let theirs_bytes = regular_content(&theirs_contents, path)?; let mode = merged_mode( read_mode(&mut base_checkout, path).await?, read_mode(&mut ours_checkout, path).await?, @@ -411,9 +452,9 @@ pub async fn plan( path, mode, merge_file( - base_bytes.as_deref(), - Some(&ours_bytes), - Some(&theirs_bytes), + base_bytes, + Some(ours_bytes), + Some(theirs_bytes), ours_name, "mainline", limits, @@ -423,17 +464,17 @@ pub async fn plan( // Modify/delete in either direction. (Some(FileKind::Regular), Some(FileKind::Regular), None) | (Some(FileKind::Regular), None, Some(FileKind::Regular)) => { - let base_bytes = read_regular(&mut base_checkout, path).await?; + let base_bytes = regular_content(&base_contents, path)?; let (ours_bytes, theirs_bytes, mode) = if f.is_some() { ( - Some(read_regular(&mut ours_checkout, path).await?), + Some(regular_content(&ours_contents, path)?), None, read_mode(&mut ours_checkout, path).await?, ) } else { ( None, - Some(read_regular(&mut theirs_checkout, path).await?), + Some(regular_content(&theirs_contents, path)?), read_mode(&mut theirs_checkout, path).await?, ) }; @@ -442,9 +483,9 @@ pub async fn plan( path, mode, merge_file( - Some(&base_bytes), - ours_bytes.as_deref(), - theirs_bytes.as_deref(), + Some(base_bytes), + ours_bytes, + theirs_bytes, ours_name, "mainline", limits, @@ -532,63 +573,6 @@ pub(crate) fn namespace_of(path: &Path) -> Result { .map_err(EngineError::fs("host path to namespace")) } -/// Whole content of a regular file at `path` in `checkout`. -pub(crate) async fn read_regular(checkout: &mut LocalCheckout, path: &Path) -> Result> { - let cancel = CancellationToken::new(); - let namespace = namespace_of(path)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - let record = lookup - .record - .ok_or_else(|| EngineError::Fs(format!("{}: absent", path.display())))?; - read_regular_record(checkout, &namespace, record, path).await -} - -async fn read_regular_record( - checkout: &mut LocalCheckout, - namespace: &NamespacePath, - record: FileRecord, - display_path: &Path, -) -> Result> { - let length = match record.payload { - acyclic_fs::kernel::FilePayload::InlineRegular(inline) => inline.as_bytes().len() as u64, - acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, - _ => { - return Err(EngineError::Fs(format!( - "{}: not a regular file", - display_path.display() - ))) - } - }; - let cancel = CancellationToken::new(); - let limits = checkout.volume_config().limits; - let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); - let mut out = Vec::with_capacity(usize::try_from(length).unwrap_or(0)); - let mut offset = 0; - while offset < length { - let take = chunk.min(length - offset); - let read = checkout - .read_file_range( - namespace, - ByteRange { - offset, - length: take, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("read file range"))? - .value; - out.extend_from_slice(&read.bytes); - offset += take; - } - Ok(out) -} - /// Contents of regular files in one generation, preserving request order. /// Opens and authenticates the generation once and resolves every namespace /// path in one SDK batch; absent and non-regular entries remain `None`. From 53094d70393dbeeea3fe98bfde9215081a2cefce Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:44:51 +0100 Subject: [PATCH 069/106] Batch merge metadata reads --- crates/acyclic/src/merge.rs | 81 ++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index ad2cba3..6efc253 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -328,6 +328,42 @@ async fn regular_contents( .collect() } +async fn record_modes( + store: &Store, + generation: GenerationId, + records: &BTreeMap, +) -> Result>> { + if records.is_empty() { + return Ok(BTreeMap::new()); + } + let checkout = store.checkout_exact(generation).await?; + let reader = checkout + .pinned_reader() + .map_err(EngineError::fs("open pinned reader"))?; + let metadata = reader + .read_record_metadata_batch( + &records.values().copied().collect::>(), + FILE_READ_CONCURRENCY, + WorkCounters::UNBOUNDED, + &CancellationToken::new(), + ) + .await + .map_err(EngineError::fs("batch read metadata"))? + .value; + Ok(records + .keys() + .cloned() + .zip(metadata) + .map(|(path, metadata)| { + let mode = match metadata.posix_mode { + MetadataField::Value(mode) => Some(mode & 0o7777), + MetadataField::Unavailable => None, + }; + (path, mode) + }) + .collect()) +} + fn regular_content<'a>(contents: &'a BTreeMap>, path: &Path) -> Result<&'a [u8]> { contents .get(path) @@ -335,6 +371,10 @@ fn regular_content<'a>(contents: &'a BTreeMap>, path: &Path) -> .ok_or_else(|| EngineError::Fs(format!("{}: regular content is absent", path.display()))) } +fn mode_at(modes: &BTreeMap>, path: &Path) -> Option { + modes.get(path).copied().flatten() +} + async fn merge_inputs( store: &Store, base: GenerationId, @@ -378,16 +418,15 @@ pub async fn plan( theirs_changed, } = inputs; - let (base_contents, theirs_contents, ours_contents) = tokio::try_join!( + let (base_contents, theirs_contents, ours_contents, base_modes, theirs_modes, ours_modes) = tokio::try_join!( regular_contents(store, base, &base_map), regular_contents(store, theirs, &theirs_map), regular_contents(store, ours, &ours_map), + record_modes(store, base, &base_map), + record_modes(store, theirs, &theirs_map), + record_modes(store, ours, &ours_map), )?; - let mut base_checkout = store.checkout_exact(base).await?; - let mut theirs_checkout = store.checkout_exact(theirs).await?; - let mut ours_checkout = store.checkout_exact(ours).await?; - let mut plan = MergePlan::default(); let mut decided_root: Option = None; let union: BTreeSet<&PathBuf> = ours_changed.iter().chain(theirs_changed.iter()).collect(); @@ -443,9 +482,9 @@ pub async fn plan( let ours_bytes = regular_content(&ours_contents, path)?; let theirs_bytes = regular_content(&theirs_contents, path)?; let mode = merged_mode( - read_mode(&mut base_checkout, path).await?, - read_mode(&mut ours_checkout, path).await?, - read_mode(&mut theirs_checkout, path).await?, + mode_at(&base_modes, path), + mode_at(&ours_modes, path), + mode_at(&theirs_modes, path), ); push_content( &mut plan, @@ -469,13 +508,13 @@ pub async fn plan( ( Some(regular_content(&ours_contents, path)?), None, - read_mode(&mut ours_checkout, path).await?, + mode_at(&ours_modes, path), ) } else { ( None, Some(regular_content(&theirs_contents, path)?), - read_mode(&mut theirs_checkout, path).await?, + mode_at(&theirs_modes, path), ) }; push_content( @@ -667,28 +706,6 @@ pub async fn read_files( Ok(contents) } -async fn read_mode(checkout: &mut LocalCheckout, path: &Path) -> Result> { - let cancel = CancellationToken::new(); - let namespace = namespace_of(path)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - if lookup.record.is_none() { - return Ok(None); - } - let metadata = checkout - .read_metadata(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("read metadata"))? - .value; - Ok(match metadata.posix_mode { - MetadataField::Value(mode) => Some(mode & 0o7777), - MetadataField::Unavailable => None, - }) -} - // --------------------------------------------------------------------------- // Writing entries into a checkout (M, R, and fork rebases) // --------------------------------------------------------------------------- From 20f11bada10d479f24a9b85fb7c927f451c21835 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:53:23 +0100 Subject: [PATCH 070/106] Batch merge parent lookups --- crates/acyclic/src/merge.rs | 64 +++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 6efc253..ca234b9 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -726,11 +726,15 @@ pub async fn apply_entries( entries: &[(PathBuf, Entry)], ) -> Result<()> { let cancel = CancellationToken::new(); - for (path, entry) in entries { - let namespace = namespace_of(path)?; + let namespaces = entries + .iter() + .map(|(path, _)| namespace_of(path)) + .collect::>>()?; + ensure_entry_parents(dst, &namespaces, &cancel).await?; + for ((_, entry), namespace) in entries.iter().zip(&namespaces) { // Boxed per entry: keeps the facade's large futures off the caller's // stack frame. - Box::pin(apply_entry(store, dst, &namespace, entry, &cancel)).await?; + Box::pin(apply_entry(store, dst, namespace, entry, &cancel)).await?; } Ok(()) } @@ -747,7 +751,6 @@ async fn apply_entry( match entry { Entry::Regular { bytes, mode } => { remove_subtree(dst, namespace, cancel).await?; - ensure_parents(dst, namespace, cancel).await?; dst.create_file( namespace.clone(), Bytes::copy_from_slice(bytes), @@ -774,7 +777,6 @@ async fn apply_entry( Entry::FromGeneration { generation } => { let mut src = store.checkout_exact(*generation).await?; remove_subtree(dst, namespace, cancel).await?; - ensure_parents(dst, namespace, cancel).await?; copy_node(&mut src, dst, namespace, cancel).await?; } } @@ -784,25 +786,47 @@ async fn apply_entry( } /// Creates missing ancestor directories of `namespace` in `dst`. -async fn ensure_parents( +async fn ensure_entry_parents( dst: &mut LocalCheckout, - namespace: &NamespacePath, + namespaces: &[NamespacePath], cancel: &CancellationToken, ) -> Result<()> { let limits = dst.volume_config().limits; - let components = namespace.components(); - for depth in 1..components.len() { - let parent = NamespacePath::new(components.iter().take(depth).cloned().collect(), limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}")))?; - let lookup = dst - .lookup_no_follow(&parent, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - if lookup.record.is_none() { - dst.create_directory(parent, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("create directory"))?; + let mut parents = BTreeSet::new(); + for namespace in namespaces { + let components = namespace.components(); + for depth in 1..components.len() { + parents.insert( + NamespacePath::new(components.iter().take(depth).cloned().collect(), limits) + .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}")))?, + ); + } + } + let parents = parents.into_iter().collect::>(); + if parents.is_empty() { + return Ok(()); + } + let lookups = dst + .lookup_batch_no_follow(&parents, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("batch parent lookup"))? + .value; + for (parent, lookup) in parents.into_iter().zip(lookups.entries) { + match lookup.record { + Some(record) if record.kind == FileKind::Directory => {} + Some(_) => { + return Err(EngineError::Fs(format!( + "{}: parent is not a directory", + acyclic_fs::namespace_to_host_path(&parent) + .map_err(EngineError::fs("resolve parent path"))? + .display() + ))) + } + None => { + dst.create_directory(parent, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("create directory"))?; + } } } Ok(()) From bf26334988f54a43663e83cc4b7a08e02add2f45 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 12:56:04 +0100 Subject: [PATCH 071/106] Batch subtree copy frontiers --- crates/acyclic/src/merge.rs | 167 +++++++++++++++++++++--------------- 1 file changed, 96 insertions(+), 71 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index ca234b9..6cc3495 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -923,106 +923,131 @@ async fn copy_node( cancel: &CancellationToken, ) -> Result<()> { let limits = src.volume_config().limits; - let mut queue: Vec = vec![namespace.clone()]; - while let Some(path) = queue.pop() { + let mut frontier = vec![namespace.clone()]; + while !frontier.is_empty() { let lookup = src - .lookup_no_follow(&path, WorkCounters::UNBOUNDED, cancel) + .lookup_batch_no_follow(&frontier, WorkCounters::UNBOUNDED, cancel) .await - .map_err(EngineError::fs("lookup"))? + .map_err(EngineError::fs("batch subtree lookup"))? .value; - let Some(record) = lookup.record else { - continue; - }; - let metadata = src - .read_metadata(&path, WorkCounters::UNBOUNDED, cancel) + let nodes = frontier + .into_iter() + .zip(lookup.entries) + .filter_map(|(path, entry)| entry.record.map(|record| (path, record))) + .collect::>(); + if nodes.is_empty() { + break; + } + let reader = src + .pinned_reader() + .map_err(EngineError::fs("open pinned reader"))?; + let metadata = reader + .read_record_metadata_batch( + &nodes.iter().map(|(_, record)| *record).collect::>(), + FILE_READ_CONCURRENCY, + WorkCounters::UNBOUNDED, + cancel, + ) .await - .map_err(EngineError::fs("read metadata"))? + .map_err(EngineError::fs("batch subtree metadata"))? .value; - match record.kind { - FileKind::Regular => { - let bytes = read_regular_ns(src, &path, &record.payload, cancel).await?; - dst.create_file( - path.clone(), - Bytes::from(bytes), - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("create file"))?; - } - FileKind::SymbolicLink => { - let target = src - .read_symbolic_link(&path, WorkCounters::UNBOUNDED, cancel) + frontier = Vec::new(); + for ((path, record), metadata) in nodes.into_iter().zip(metadata) { + match record.kind { + FileKind::Regular => { + let bytes = read_regular_record(&reader, record, cancel).await?; + dst.create_file( + path.clone(), + Bytes::from(bytes), + WorkCounters::UNBOUNDED, + cancel, + ) .await - .map_err(EngineError::fs("read symlink"))? - .value; - dst.create_symbolic_link( - path.clone(), - Bytes::copy_from_slice(&target), - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("create symlink"))?; - } - FileKind::Directory => { - dst.create_directory(path.clone(), WorkCounters::UNBOUNDED, cancel) + .map_err(EngineError::fs("create file"))?; + } + FileKind::SymbolicLink => { + let target = src + .read_symbolic_link(&path, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("read symlink"))? + .value; + dst.create_symbolic_link( + path.clone(), + Bytes::copy_from_slice(&target), + WorkCounters::UNBOUNDED, + cancel, + ) .await - .map_err(EngineError::fs("create directory"))?; - for name in list_children(src, &path, cancel).await? { - queue.push(child_path(&path, &name, limits)?); + .map_err(EngineError::fs("create symlink"))?; + } + FileKind::Directory => { + dst.create_directory(path.clone(), WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("create directory"))?; + for name in list_children(src, &path, cancel).await? { + frontier.push(child_path(&path, &name, limits)?); + } + } + other => { + return Err(EngineError::Fs(format!( + "cannot copy a {other:?} node (only files, symlinks, and directories)" + ))) } } - other => { - return Err(EngineError::Fs(format!( - "cannot copy a {other:?} node (only files, symlinks, and directories)" - ))) + if let MetadataField::Value(mode) = metadata.posix_mode { + let set = FileMetadata { + posix_mode: MetadataField::Value(mode), + ..FileMetadata::default() + }; + dst.set_metadata(path, set, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("set metadata"))?; } } - if let MetadataField::Value(mode) = metadata.posix_mode { - let set = FileMetadata { - posix_mode: MetadataField::Value(mode), - ..FileMetadata::default() - }; - dst.set_metadata(path, set, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("set metadata"))?; - } } Ok(()) } -async fn read_regular_ns( - checkout: &mut LocalCheckout, - namespace: &NamespacePath, - payload: &acyclic_fs::kernel::FilePayload, +async fn read_regular_record( + reader: &acyclic_fs::PinnedReader, + record: FileRecord, cancel: &CancellationToken, -) -> Result> { - let length = match payload { +) -> Result> +where + A: acyclic_fs::AsyncAuthorityStore, + O: acyclic_fs::AsyncObjectStore, +{ + let length = match record.payload { acyclic_fs::kernel::FilePayload::InlineRegular(inline) => inline.as_bytes().len() as u64, - acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => *logical_bytes, + acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, _ => return Err(EngineError::Fs("regular file with foreign payload".into())), }; - let limits = checkout.volume_config().limits; - let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); + let chunk = TRANSFER_BYTES; let mut out = Vec::with_capacity(usize::try_from(length).unwrap_or(0)); let mut offset = 0; while offset < length { let take = chunk.min(length - offset); - let read = checkout - .read_file_range( - namespace, - ByteRange { - offset, - length: take, - }, + let read = reader + .read_file_record_ranges( + &[FileRecordRangeReadRequest { + record, + range: ByteRange { + offset, + length: take, + }, + }], + 1, WorkCounters::UNBOUNDED, cancel, ) .await .map_err(EngineError::fs("read file range"))? .value; - out.extend_from_slice(&read.bytes); + let chunk = read + .into_iter() + .next() + .ok_or_else(|| EngineError::Fs("record range read returned no result".into()))?; + out.extend_from_slice(&chunk.bytes); offset += take; } Ok(out) From dd577ba8f4bd3790090e39b182a89fc53eb278f6 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:03:53 +0100 Subject: [PATCH 072/106] Reuse resolved symlink records --- crates/acyclic/src/merge.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 6cc3495..8b8de26 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -281,9 +281,14 @@ async fn records_at( .map(|path| namespace_of(path)) .collect::>>()?; let records = generation - .lookup_paths(&namespaces) + .lookup_paths( + &namespaces, + WorkCounters::UNBOUNDED, + &CancellationToken::new(), + ) .await - .map_err(EngineError::fs("lookup merge paths"))?; + .map_err(EngineError::fs("lookup merge paths"))? + .value; Ok(paths .iter() .cloned() @@ -966,10 +971,10 @@ async fn copy_node( .map_err(EngineError::fs("create file"))?; } FileKind::SymbolicLink => { - let target = src - .read_symbolic_link(&path, WorkCounters::UNBOUNDED, cancel) + let target = reader + .read_symbolic_link_record(record, WorkCounters::UNBOUNDED, cancel) .await - .map_err(EngineError::fs("read symlink"))? + .map_err(EngineError::fs("read resolved symlink"))? .value; dst.create_symbolic_link( path.clone(), From 6b761fd1c479691c9aae2bd15f8a6de79d8bdd0e Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:07:44 +0100 Subject: [PATCH 073/106] Batch subtree reads and writes --- crates/acyclic/src/merge.rs | 134 ++++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 60 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 8b8de26..eb381f9 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -22,7 +22,8 @@ use acyclic_fs::text_merge::{ merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, }; use acyclic_fs::{ - ByteRange, CancellationToken, FileRecordRangeReadRequest, GenerationId, WorkCounters, + AuthoredMutation, ByteRange, CancellationToken, FileRecordRangeReadRequest, GenerationId, + WorkCounters, }; use bytes::Bytes; @@ -956,19 +957,20 @@ async fn copy_node( .await .map_err(EngineError::fs("batch subtree metadata"))? .value; + let regular = read_regular_frontier(&reader, &nodes, cancel).await?; frontier = Vec::new(); - for ((path, record), metadata) in nodes.into_iter().zip(metadata) { + let mut mutations = Vec::with_capacity(nodes.len()); + for (index, ((path, record), metadata)) in nodes.into_iter().zip(metadata).enumerate() { match record.kind { FileKind::Regular => { - let bytes = read_regular_record(&reader, record, cancel).await?; - dst.create_file( - path.clone(), - Bytes::from(bytes), - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("create file"))?; + let bytes = regular.get(index).ok_or_else(|| { + EngineError::Fs("subtree read omitted a regular file".into()) + })?; + mutations.push(AuthoredMutation::CreateFile { + path, + bytes: Bytes::from(bytes.clone()), + metadata, + }); } FileKind::SymbolicLink => { let target = reader @@ -976,19 +978,17 @@ async fn copy_node( .await .map_err(EngineError::fs("read resolved symlink"))? .value; - dst.create_symbolic_link( - path.clone(), - Bytes::copy_from_slice(&target), - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("create symlink"))?; + mutations.push(AuthoredMutation::CreateSymbolicLink { + path, + target, + metadata, + }); } FileKind::Directory => { - dst.create_directory(path.clone(), WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("create directory"))?; + mutations.push(AuthoredMutation::CreateDirectory { + path: path.clone(), + metadata, + }); for name in list_children(src, &path, cancel).await? { frontier.push(child_path(&path, &name, limits)?); } @@ -999,63 +999,77 @@ async fn copy_node( ))) } } - if let MetadataField::Value(mode) = metadata.posix_mode { - let set = FileMetadata { - posix_mode: MetadataField::Value(mode), - ..FileMetadata::default() - }; - dst.set_metadata(path, set, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("set metadata"))?; - } + } + let maximum = usize::try_from(dst.volume_config().limits.maximum_mutations_per_batch) + .unwrap_or(usize::MAX) + .saturating_div(2) + .max(1); + for chunk in mutations.chunks(maximum) { + dst.apply_authored_transaction(chunk.to_vec(), WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("create subtree frontier"))?; } } Ok(()) } -async fn read_regular_record( +async fn read_regular_frontier( reader: &acyclic_fs::PinnedReader, - record: FileRecord, + nodes: &[(NamespacePath, FileRecord)], cancel: &CancellationToken, -) -> Result> +) -> Result>> where A: acyclic_fs::AsyncAuthorityStore, O: acyclic_fs::AsyncObjectStore, { - let length = match record.payload { - acyclic_fs::kernel::FilePayload::InlineRegular(inline) => inline.as_bytes().len() as u64, - acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, - _ => return Err(EngineError::Fs("regular file with foreign payload".into())), - }; - let chunk = TRANSFER_BYTES; - let mut out = Vec::with_capacity(usize::try_from(length).unwrap_or(0)); - let mut offset = 0; - while offset < length { - let take = chunk.min(length - offset); + let mut contents = vec![Vec::new(); nodes.len()]; + let mut offsets = vec![0_u64; nodes.len()]; + loop { + let mut destinations = Vec::new(); + let mut requests = Vec::new(); + for (index, ((_, record), offset)) in nodes.iter().zip(&mut offsets).enumerate() { + let length = match record.payload { + acyclic_fs::kernel::FilePayload::InlineRegular(inline) => { + inline.as_bytes().len() as u64 + } + acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, + _ => continue, + }; + if *offset >= length { + continue; + } + let take = TRANSFER_BYTES.min(length - *offset); + destinations.push(index); + requests.push(FileRecordRangeReadRequest { + record: *record, + range: ByteRange { + offset: *offset, + length: take, + }, + }); + *offset += take; + } + if requests.is_empty() { + break; + } let read = reader .read_file_record_ranges( - &[FileRecordRangeReadRequest { - record, - range: ByteRange { - offset, - length: take, - }, - }], - 1, + &requests, + FILE_READ_CONCURRENCY, WorkCounters::UNBOUNDED, cancel, ) .await - .map_err(EngineError::fs("read file range"))? + .map_err(EngineError::fs("batch read subtree ranges"))? .value; - let chunk = read - .into_iter() - .next() - .ok_or_else(|| EngineError::Fs("record range read returned no result".into()))?; - out.extend_from_slice(&chunk.bytes); - offset += take; + for (index, chunk) in destinations.into_iter().zip(read) { + contents + .get_mut(index) + .ok_or_else(|| EngineError::Fs("batch read returned an invalid index".into()))? + .extend_from_slice(&chunk.bytes); + } } - Ok(out) + Ok(contents) } /// Which of `paths` the repo's `.gitignore` rules ignore, per From f579fbe999254cafba6fd5b1ebf8049da2e7f8a7 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:10:28 +0100 Subject: [PATCH 074/106] Batch subtree removals --- crates/acyclic/src/merge.rs | 58 +++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index eb381f9..87a90ef 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -889,31 +889,47 @@ async fn remove_subtree( cancel: &CancellationToken, ) -> Result<()> { let limits = dst.volume_config().limits; - // (path, children_expanded) - let mut stack: Vec<(NamespacePath, bool)> = vec![(namespace.clone(), false)]; - while let Some((path, expanded)) = stack.pop() { - if expanded { - dst.remove(path, None, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("remove"))?; - continue; - } + let mut frontier = vec![namespace.clone()]; + let mut levels = Vec::new(); + while !frontier.is_empty() { let lookup = dst - .lookup_no_follow(&path, WorkCounters::UNBOUNDED, cancel) + .lookup_batch_no_follow(&frontier, WorkCounters::UNBOUNDED, cancel) .await - .map_err(EngineError::fs("lookup"))? + .map_err(EngineError::fs("batch subtree removal lookup"))? .value; - let Some(record) = lookup.record else { - continue; - }; - if record.kind == FileKind::Directory { - let children = list_children(dst, &path, cancel).await?; - stack.push((path.clone(), true)); - for name in children { - stack.push((child_path(&path, &name, limits)?, false)); + let nodes = frontier + .into_iter() + .zip(lookup.entries) + .filter_map(|(path, entry)| entry.record.map(|record| (path, record))) + .collect::>(); + if nodes.is_empty() { + break; + } + frontier = Vec::new(); + for (path, record) in &nodes { + if record.kind == FileKind::Directory { + for name in list_children(dst, path, cancel).await? { + frontier.push(child_path(path, &name, limits)?); + } } - } else { - stack.push((path, true)); + } + levels.push(nodes); + } + let maximum = usize::try_from(dst.volume_config().limits.maximum_mutations_per_batch) + .unwrap_or(usize::MAX) + .max(1); + for level in levels.into_iter().rev() { + let removals = level + .into_iter() + .map(|(path, record)| AuthoredMutation::Remove { + path, + expected_file_id: Some(record.file_id), + }) + .collect::>(); + for chunk in removals.chunks(maximum) { + dst.apply_authored_transaction(chunk.to_vec(), WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("remove subtree frontier"))?; } } Ok(()) From a28ead9f90505d7bc61e59e7a3b6a14ad3366c5a Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:18:35 +0100 Subject: [PATCH 075/106] Batch sibling directory scans --- crates/acyclic/src/merge.rs | 101 ++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 40 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 87a90ef..af9552f 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -22,8 +22,8 @@ use acyclic_fs::text_merge::{ merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, }; use acyclic_fs::{ - AuthoredMutation, ByteRange, CancellationToken, FileRecordRangeReadRequest, GenerationId, - WorkCounters, + AuthoredMutation, ByteRange, CancellationToken, DirectoryRecordPageRequest, + FileRecordRangeReadRequest, GenerationId, WorkCounters, }; use bytes::Bytes; @@ -849,34 +849,60 @@ fn child_path( .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}"))) } -async fn list_children( - checkout: &mut LocalCheckout, - namespace: &NamespacePath, +async fn list_child_paths( + checkout: &LocalCheckout, + directories: &[NamespacePath], cancel: &CancellationToken, -) -> Result> { - let mut names = Vec::new(); - let mut after = None; - loop { - let page = checkout - .list_directory_records( - namespace, - after.as_ref(), - PAGE_ENTRIES, +) -> Result> { + if directories.is_empty() { + return Ok(Vec::new()); + } + let limits = checkout.volume_config().limits; + let reader = checkout + .pinned_reader() + .map_err(EngineError::fs("open directory reader"))?; + let mut pending = directories + .iter() + .cloned() + .map(|path| DirectoryRecordPageRequest { + path, + after: None, + maximum_entries: PAGE_ENTRIES, + }) + .collect::>(); + let mut children = Vec::new(); + while !pending.is_empty() { + let pages = reader + .list_directory_record_pages( + &pending, + FILE_READ_CONCURRENCY, WorkCounters::UNBOUNDED, cancel, ) .await - .map_err(EngineError::fs("list directory"))? + .map_err(EngineError::fs("batch list directories"))? .value; - for entry in &page.entries { - names.push(entry.name.clone()); - } - match page.entries.last() { - Some(last) if page.has_more => after = Some(last.name.clone()), - _ => break, + let mut next = Vec::new(); + for (request, page) in pending.into_iter().zip(pages) { + for entry in &page.entries { + children.push(child_path(&request.path, &entry.name, limits)?); + } + if page.has_more { + let after = page + .entries + .last() + .ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))? + .name + .clone(); + next.push(DirectoryRecordPageRequest { + after: Some(after), + ..request + }); + } } + pending = next; } - Ok(names) + Ok(children) } /// Removes `namespace` from `dst`, recursively for directories; absent is @@ -888,7 +914,6 @@ async fn remove_subtree( namespace: &NamespacePath, cancel: &CancellationToken, ) -> Result<()> { - let limits = dst.volume_config().limits; let mut frontier = vec![namespace.clone()]; let mut levels = Vec::new(); while !frontier.is_empty() { @@ -905,14 +930,12 @@ async fn remove_subtree( if nodes.is_empty() { break; } - frontier = Vec::new(); - for (path, record) in &nodes { - if record.kind == FileKind::Directory { - for name in list_children(dst, path, cancel).await? { - frontier.push(child_path(path, &name, limits)?); - } - } - } + let directories = nodes + .iter() + .filter(|(_, record)| record.kind == FileKind::Directory) + .map(|(path, _)| path.clone()) + .collect::>(); + frontier = list_child_paths(dst, &directories, cancel).await?; levels.push(nodes); } let maximum = usize::try_from(dst.volume_config().limits.maximum_mutations_per_batch) @@ -944,7 +967,6 @@ async fn copy_node( namespace: &NamespacePath, cancel: &CancellationToken, ) -> Result<()> { - let limits = src.volume_config().limits; let mut frontier = vec![namespace.clone()]; while !frontier.is_empty() { let lookup = src @@ -974,7 +996,12 @@ async fn copy_node( .map_err(EngineError::fs("batch subtree metadata"))? .value; let regular = read_regular_frontier(&reader, &nodes, cancel).await?; - frontier = Vec::new(); + let directories = nodes + .iter() + .filter(|(_, record)| record.kind == FileKind::Directory) + .map(|(path, _)| path.clone()) + .collect::>(); + frontier = list_child_paths(src, &directories, cancel).await?; let mut mutations = Vec::with_capacity(nodes.len()); for (index, ((path, record), metadata)) in nodes.into_iter().zip(metadata).enumerate() { match record.kind { @@ -1001,13 +1028,7 @@ async fn copy_node( }); } FileKind::Directory => { - mutations.push(AuthoredMutation::CreateDirectory { - path: path.clone(), - metadata, - }); - for name in list_children(src, &path, cancel).await? { - frontier.push(child_path(&path, &name, limits)?); - } + mutations.push(AuthoredMutation::CreateDirectory { path, metadata }); } other => { return Err(EngineError::Fs(format!( From 209e99ad27c092917daa9c8fb3081c1832a8bab6 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:20:42 +0100 Subject: [PATCH 076/106] Reuse merge source checkouts --- crates/acyclic/src/merge.rs | 82 ++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 46 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index af9552f..8e1e6f7 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -737,54 +737,44 @@ pub async fn apply_entries( .map(|(path, _)| namespace_of(path)) .collect::>>()?; ensure_entry_parents(dst, &namespaces, &cancel).await?; + let mut source = None; for ((_, entry), namespace) in entries.iter().zip(&namespaces) { - // Boxed per entry: keeps the facade's large futures off the caller's - // stack frame. - Box::pin(apply_entry(store, dst, namespace, entry, &cancel)).await?; - } - Ok(()) -} - -async fn apply_entry( - store: &Store, - dst: &mut LocalCheckout, - namespace: &NamespacePath, - entry: &Entry, - cancel: &CancellationToken, -) -> Result<()> { - { - { - match entry { - Entry::Regular { bytes, mode } => { - remove_subtree(dst, namespace, cancel).await?; - dst.create_file( - namespace.clone(), - Bytes::copy_from_slice(bytes), - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("create file"))?; - if let Some(mode) = mode { - let metadata = FileMetadata { - posix_mode: MetadataField::Value(*mode), - ..FileMetadata::default() - }; - dst.set_metadata( - namespace.clone(), - metadata, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("set metadata"))?; - } - } - Entry::FromGeneration { generation } => { - let mut src = store.checkout_exact(*generation).await?; - remove_subtree(dst, namespace, cancel).await?; - copy_node(&mut src, dst, namespace, cancel).await?; + match entry { + Entry::Regular { bytes, mode } => { + source = None; + remove_subtree(dst, namespace, &cancel).await?; + let metadata = FileMetadata { + posix_mode: mode.map_or(MetadataField::Unavailable, MetadataField::Value), + ..FileMetadata::default() + }; + dst.apply_authored_transaction( + vec![AuthoredMutation::CreateFile { + path: namespace.clone(), + bytes: Bytes::copy_from_slice(bytes), + metadata, + }], + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(EngineError::fs("create merge file"))?; + } + Entry::FromGeneration { generation } => { + let needs_checkout = + source + .as_ref() + .is_none_or(|(current, _): &(GenerationId, LocalCheckout)| { + current != generation + }); + if needs_checkout { + source = Some((*generation, store.checkout_exact(*generation).await?)); } + let src = &mut source + .as_mut() + .ok_or_else(|| EngineError::Fs("source checkout missing".into()))? + .1; + remove_subtree(dst, namespace, &cancel).await?; + copy_node(src, dst, namespace, &cancel).await?; } } } From ab12c9d1812c23b518c7f54ce90f6623513c660b Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:25:55 +0100 Subject: [PATCH 077/106] Avoid metadata reads while walking subtrees --- crates/acyclic/src/merge.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 8e1e6f7..cee9444 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -22,7 +22,7 @@ use acyclic_fs::text_merge::{ merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, }; use acyclic_fs::{ - AuthoredMutation, ByteRange, CancellationToken, DirectoryRecordPageRequest, + AuthoredMutation, ByteRange, CancellationToken, DirectoryPageRequest, FileRecordRangeReadRequest, GenerationId, WorkCounters, }; use bytes::Bytes; @@ -854,7 +854,7 @@ async fn list_child_paths( let mut pending = directories .iter() .cloned() - .map(|path| DirectoryRecordPageRequest { + .map(|path| DirectoryPageRequest { path, after: None, maximum_entries: PAGE_ENTRIES, @@ -863,7 +863,7 @@ async fn list_child_paths( let mut children = Vec::new(); while !pending.is_empty() { let pages = reader - .list_directory_record_pages( + .list_directory_pages( &pending, FILE_READ_CONCURRENCY, WorkCounters::UNBOUNDED, @@ -884,7 +884,7 @@ async fn list_child_paths( .ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))? .name .clone(); - next.push(DirectoryRecordPageRequest { + next.push(DirectoryPageRequest { after: Some(after), ..request }); From e084115d847343ec5cf7eab8ad88c292d7969d4a Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:30:03 +0100 Subject: [PATCH 078/106] Keep session hooks scan free --- crates/acyclic/src/pipeline.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 96f764d..403db82 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -279,6 +279,8 @@ impl Request { Self::Status { .. } | Self::TurnStarted { .. } | Self::RecordGeneration { .. } + | Self::SessionStarted { .. } + | Self::SessionEnded { .. } | Self::Shutdown { .. } ) } @@ -1304,7 +1306,10 @@ impl Pipeline { let result = async { self.index.session_started(&session_id, &host)?; // Session start is a coarse boundary. - self.commit_engine().await + if self.state == State::Ready { + self.commit_engine().await?; + } + Ok(()) } .await; let _ = reply.send(result); @@ -1313,7 +1318,10 @@ impl Pipeline { Request::SessionEnded { session_id, reply } => { let result = async { self.index.session_ended(&session_id)?; - self.commit_engine().await + if self.state == State::Ready { + self.commit_engine().await?; + } + Ok(()) } .await; let _ = reply.send(result); @@ -2382,6 +2390,21 @@ mod readiness_tests { } .requires_ready()); + let (started_reply, _) = oneshot::channel(); + assert!(!Request::SessionStarted { + session_id: "session".to_owned(), + host: "host".to_owned(), + reply: started_reply, + } + .requires_ready()); + + let (ended_reply, _) = oneshot::channel(); + assert!(!Request::SessionEnded { + session_id: "session".to_owned(), + reply: ended_reply, + } + .requires_ready()); + let (checkpoint_reply, _) = oneshot::channel(); assert!(Request::Checkpoint { kind: CheckpointKind::Manual, From 45b8ce3ab59759dad7b5769069c0060d7e0b8b64 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:47:48 +0100 Subject: [PATCH 079/106] Use SDK durable rename Signed-off-by: Var1377 --- Cargo.lock | 1 - crates/acyclic/src/rewind.rs | 47 +----------------------------------- 2 files changed, 1 insertion(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77da099..ecdbe4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,7 +89,6 @@ dependencies = [ "prost-types", "thiserror", "tokio", - "windows-sys 0.61.2", ] [[package]] diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index b0d349c..e703e8b 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -826,53 +826,8 @@ fn publish_locator(repo: &Path, journal: &Path) -> Result { Ok(path) } -#[cfg(unix)] -fn durable_rename(from: &Path, to: &Path, _replace: bool) -> std::io::Result<()> { - std::fs::rename(from, to)?; - let parent = to - .parent() - .ok_or_else(|| std::io::Error::other("rename path has no parent"))?; - std::fs::File::open(parent)?.sync_all()?; - if from.parent() != to.parent() { - let parent = from - .parent() - .ok_or_else(|| std::io::Error::other("rename path has no parent"))?; - std::fs::File::open(parent)?.sync_all()?; - } - Ok(()) -} - -#[cfg(windows)] -#[allow( - unsafe_code, - reason = "MoveFileExW receives two terminated UTF-16 path buffers" -)] fn durable_rename(from: &Path, to: &Path, replace: bool) -> std::io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::{ - MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, - }; - let from: Vec = from - .as_os_str() - .encode_wide() - .chain(std::iter::once(0)) - .collect(); - let to: Vec = to - .as_os_str() - .encode_wide() - .chain(std::iter::once(0)) - .collect(); - let flags = MOVEFILE_WRITE_THROUGH - | if replace { - MOVEFILE_REPLACE_EXISTING - } else { - 0 - }; - if unsafe { MoveFileExW(from.as_ptr(), to.as_ptr(), flags) } == 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) - } + acyclic_fs::durable_rename(from, to, replace) } #[cfg(unix)] From 2c9cf53b1e254b5f20f55443b810cbbf89754bbb Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 13:53:22 +0100 Subject: [PATCH 080/106] Reuse SDK durable rename for continuity Signed-off-by: Var1377 --- crates/acyclic/src/store.rs | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index 3e280f0..d0243cc 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -3,9 +3,9 @@ //! Everything lives OUTSIDE the working tree (capture snapshots the whole //! tree and fail-closes on sockets). The repo carries only `.acyclic/config.toml`. -use std::path::{Path, PathBuf}; #[cfg(target_os = "windows")] -use std::{fs::OpenOptions, io::Write}; +use std::io::Write; +use std::path::{Path, PathBuf}; use acyclic_fs::model::{ AccessMode, CheckoutMode, ConsistencyMode, GenerationSelector, Lifecycle, MutationMode, @@ -469,7 +469,7 @@ fn atomic_write_json(path: &Path, value: &T) -> Result<()> { #[cfg(target_os = "windows")] pub(crate) fn durable_replace(path: &Path, bytes: &[u8]) -> Result<()> { let tmp = path.with_extension("bin.tmp"); - let mut file = OpenOptions::new() + let mut file = std::fs::OpenOptions::new() .write(true) .create(true) .truncate(true) @@ -477,32 +477,7 @@ pub(crate) fn durable_replace(path: &Path, bytes: &[u8]) -> Result<()> { file.write_all(bytes)?; file.sync_all()?; drop(file); - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::{ - MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, - }; - let from: Vec = tmp - .as_os_str() - .encode_wide() - .chain(std::iter::once(0)) - .collect(); - let to: Vec = path - .as_os_str() - .encode_wide() - .chain(std::iter::once(0)) - .collect(); - // SAFETY: both paths are terminated UTF-16 buffers live for this call. - #[allow(unsafe_code)] - let replaced = unsafe { - MoveFileExW( - from.as_ptr(), - to.as_ptr(), - MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, - ) - }; - if replaced == 0 { - return Err(std::io::Error::last_os_error().into()); - } + acyclic_fs::durable_rename(&tmp, path, true)?; Ok(()) } From 8a33b069f508c5b702b43d1c6b7b5013d567b4db Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 14:10:33 +0100 Subject: [PATCH 081/106] Keep status constant time Signed-off-by: Var1377 --- crates/acyclic/src/server.rs | 55 +----------------------------------- 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index cfa7cf7..ad75b6f 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -209,7 +209,6 @@ pub fn run(repo_root: &Path) -> Result<(), String> { mounts, handle: handle.clone(), index_db: paths.index_db(), - store_root: paths.root.clone(), repo_root, config, shutdown: shutdown.clone(), @@ -222,7 +221,6 @@ pub fn run(repo_root: &Path) -> Result<(), String> { spec, live_sessions: Arc::new(Mutex::new(HashSet::new())), last_activity: Arc::new(Mutex::new(Instant::now())), - store_bytes: Arc::new(Mutex::new(None)), }; runtime.block_on(serve_until_done(server, listener, shutdown, handle)); @@ -242,7 +240,6 @@ struct Server { mounts: MountCapability, handle: PipelineHandle, index_db: PathBuf, - store_root: PathBuf, repo_root: PathBuf, config: Config, shutdown: Arc, @@ -257,9 +254,6 @@ struct Server { live_sessions: Arc>>, /// When the last request arrived; the idle-exit clock. last_activity: Arc>, - /// `store size` for `status`, refreshed in the background: walking the - /// object directory costs ~1s on an aged store. - store_bytes: Arc>>, } impl Server { @@ -337,32 +331,6 @@ impl Server { && self.forks.lock().await.is_empty() } - /// The store's size on disk, from a cache that a background walk - /// refreshes once it is a minute old. Only the very first call walks. - async fn store_size(&self) -> u64 { - const FRESH: Duration = Duration::from_secs(60); - let root = self.store_root.join("store"); - let cached = *self.store_bytes.lock().await; - match cached { - Some((at, bytes)) if at.elapsed() < FRESH => bytes, - Some((_, bytes)) => { - let cache = Arc::clone(&self.store_bytes); - tokio::task::spawn_blocking(move || { - let fresh = directory_bytes(&root); - if let Ok(mut slot) = cache.try_lock() { - *slot = Some((Instant::now(), fresh)); - } - }); - bytes - } - None => { - let bytes = tokio::task::block_in_place(|| directory_bytes(&root)); - *self.store_bytes.lock().await = Some((Instant::now(), bytes)); - bytes - } - } - } - #[allow( clippy::too_many_lines, clippy::cognitive_complexity, @@ -381,7 +349,7 @@ impl Server { state: format!("{:?}", status.state).to_lowercase(), last_checkpoint: status.last_checkpoint, unpublished: status.unpublished, - store_bytes: self.store_size().await, + store_bytes: 0, repo_root: self.repo_root.display().to_string(), mount_provider: self.mounts.provider.to_owned(), mount_available: self.mounts.available, @@ -2019,24 +1987,3 @@ fn timeline_entry(row: CheckpointRow) -> proto::TimelineEntry { fn hex_generation(generation: acyclic::GenerationId) -> String { acyclic::generation_hex(generation) } - -fn directory_bytes(root: &Path) -> u64 { - let mut total = 0u64; - let mut stack = vec![root.to_path_buf()]; - while let Some(dir) = stack.pop() { - let Ok(entries) = std::fs::read_dir(&dir) else { - continue; - }; - for entry in entries.flatten() { - let Ok(metadata) = entry.metadata() else { - continue; - }; - if metadata.is_dir() { - stack.push(entry.path()); - } else { - total += metadata.len(); - } - } - } - total -} From f7837456d7e5206c4f885e0eb29d2e26c9310a1b Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 14:23:36 +0100 Subject: [PATCH 082/106] Use exact SDK rename semantics Signed-off-by: Var1377 --- crates/acyclic/src/install.rs | 2 +- crates/acyclic/src/rewind.rs | 22 +++++++++------------- crates/acyclic/src/store.rs | 6 ++---- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/crates/acyclic/src/install.rs b/crates/acyclic/src/install.rs index 027c8de..1698d07 100644 --- a/crates/acyclic/src/install.rs +++ b/crates/acyclic/src/install.rs @@ -677,7 +677,7 @@ fn write_atomic(path: &Path, text: &str) -> Result<(), String> { file.write_all(text.as_bytes()).map_err(stringify)?; file.sync_all().map_err(stringify)?; drop(file); - std::fs::rename(&tmp, path).map_err(stringify) + acyclic_fs::durable_rename(&tmp, path, acyclic_fs::RenameMode::Replace).map_err(stringify) } /// Where a host reads its MCP config. This decides both the path and the diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index e703e8b..4d71db3 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -7,8 +7,8 @@ use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use acyclic_fs::{ - exchange_native_entries, materialize_checkout, materialize_checkout_host_path, - publish_native_exchange, MaterializeError, MaterializeOptions, + durable_rename, exchange_native_entries, materialize_checkout, materialize_checkout_host_path, + publish_native_exchange, MaterializeError, MaterializeOptions, RenameMode, }; use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; use serde::{Deserialize, Serialize}; @@ -171,7 +171,7 @@ pub(crate) async fn materialize_path_into_checkout( }, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { let renamed = match replace { - PathReplace::Atomic => durable_rename(&staged, &destination, false), + PathReplace::Atomic => durable_rename(&staged, &destination, RenameMode::NoReplace), PathReplace::LiveMount => std::fs::rename(&staged, &destination), }; if let Err(error) = renamed { @@ -633,7 +633,7 @@ fn park_replaced_tree(tmp: &Path, parent: &Path, name: &str) -> Result PARK_SEQUENCE.fetch_add(1, Ordering::Relaxed) ); let sibling = parent.join(format!(".{name}.{}-trash-{unique}", crate::product::NAME)); - durable_rename(tmp, &sibling, false).map_err(|error| { + durable_rename(tmp, &sibling, RenameMode::NoReplace).map_err(|error| { EngineError::Restore(format!( "rewind: park the replaced tree at {}: {error}", sibling.display() @@ -705,11 +705,11 @@ pub fn recover(journal_path: &Path) -> Result> { .as_deref() .ok_or_else(|| EngineError::Restore("rewind scratch path is missing".into()))?; move_back(&journal.tmp, old, &journal.carried)?; - durable_rename(old, &journal.repo_root, false)?; + durable_rename(old, &journal.repo_root, RenameMode::NoReplace)?; } else if !repo_present && tmp_present { // A journal from an older implementation can have no scratch. // Make the repo whole before attempting store startup. - durable_rename(&journal.tmp, &journal.repo_root, false)?; + durable_rename(&journal.tmp, &journal.repo_root, RenameMode::NoReplace)?; } else { // If the exchange never started, carried paths are still in // tmp and must return to the live tree. After a completed @@ -718,7 +718,7 @@ pub fn recover(journal_path: &Path) -> Result> { } #[cfg(not(windows))] if !repo_present && tmp_present { - durable_rename(&journal.tmp, &journal.repo_root, false)?; + durable_rename(&journal.tmp, &journal.repo_root, RenameMode::NoReplace)?; } else { move_back(&journal.tmp, &journal.repo_root, &journal.carried)?; } @@ -792,7 +792,7 @@ fn write_journal(path: &Path, journal: &Journal) -> Result<()> { file.write_all(text.as_bytes())?; file.sync_all()?; drop(file); - durable_rename(&tmp, path, true) + durable_rename(&tmp, path, RenameMode::Replace) }; write().map_err(|error| { let _ = std::fs::remove_file(&tmp); @@ -810,7 +810,7 @@ fn write_locator(path: &Path, locator: &RecoveryLocator) -> Result<()> { file.write_all(&text)?; file.sync_all()?; drop(file); - durable_rename(&tmp, path, true)?; + durable_rename(&tmp, path, RenameMode::Replace)?; Ok(()) } @@ -826,10 +826,6 @@ fn publish_locator(repo: &Path, journal: &Path) -> Result { Ok(path) } -fn durable_rename(from: &Path, to: &Path, replace: bool) -> std::io::Result<()> { - acyclic_fs::durable_rename(from, to, replace) -} - #[cfg(unix)] fn sync_parent(path: &Path) -> Result<()> { std::fs::File::open( diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index d0243cc..20c98f4 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -3,7 +3,6 @@ //! Everything lives OUTSIDE the working tree (capture snapshots the whole //! tree and fail-closes on sockets). The repo carries only `.acyclic/config.toml`. -#[cfg(target_os = "windows")] use std::io::Write; use std::path::{Path, PathBuf}; @@ -462,11 +461,10 @@ fn atomic_write_json(path: &Path, value: &T) -> Result<()> { .map_err(|error| EngineError::Store(format!("encode {}: {error}", path.display())))?; let tmp = path.with_extension("json.tmp"); std::fs::write(&tmp, text)?; - std::fs::rename(&tmp, path)?; + acyclic_fs::durable_rename(&tmp, path, acyclic_fs::RenameMode::Replace)?; Ok(()) } -#[cfg(target_os = "windows")] pub(crate) fn durable_replace(path: &Path, bytes: &[u8]) -> Result<()> { let tmp = path.with_extension("bin.tmp"); let mut file = std::fs::OpenOptions::new() @@ -477,7 +475,7 @@ pub(crate) fn durable_replace(path: &Path, bytes: &[u8]) -> Result<()> { file.write_all(bytes)?; file.sync_all()?; drop(file); - acyclic_fs::durable_rename(&tmp, path, true)?; + acyclic_fs::durable_rename(&tmp, path, acyclic_fs::RenameMode::Replace)?; Ok(()) } From 814464874216f13eb72c1f4b08e147344027122b Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 14:43:54 +0100 Subject: [PATCH 083/106] Use generation safe pinned reads --- crates/acyclic/src/merge.rs | 119 ++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 58 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index cee9444..65894ba 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -22,8 +22,8 @@ use acyclic_fs::text_merge::{ merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, }; use acyclic_fs::{ - AuthoredMutation, ByteRange, CancellationToken, DirectoryPageRequest, - FileRecordRangeReadRequest, GenerationId, WorkCounters, + AuthoredMutation, ByteRange, CancellationToken, DirectoryPageRequest, FileRangeReadRequest, + GenerationId, WorkCounters, }; use bytes::Bytes; @@ -346,13 +346,12 @@ async fn record_modes( let reader = checkout .pinned_reader() .map_err(EngineError::fs("open pinned reader"))?; + let paths = records + .keys() + .map(|path| namespace_of(path)) + .collect::>>()?; let metadata = reader - .read_record_metadata_batch( - &records.values().copied().collect::>(), - FILE_READ_CONCURRENCY, - WorkCounters::UNBOUNDED, - &CancellationToken::new(), - ) + .describe_files(&paths, WorkCounters::UNBOUNDED, &CancellationToken::new()) .await .map_err(EngineError::fs("batch read metadata"))? .value; @@ -360,11 +359,11 @@ async fn record_modes( .keys() .cloned() .zip(metadata) - .map(|(path, metadata)| { - let mode = match metadata.posix_mode { + .map(|(path, description)| { + let mode = description.and_then(|description| match description.metadata.posix_mode { MetadataField::Value(mode) => Some(mode & 0o7777), MetadataField::Unavailable => None, - }; + }); (path, mode) }) .collect()) @@ -626,7 +625,7 @@ pub async fn read_files( generation: GenerationId, paths: &[PathBuf], ) -> Result>>> { - let mut checkout = store.checkout_exact(generation).await?; + let checkout = store.checkout_exact(generation).await?; let namespaces = paths .iter() .map(|path| namespace_of(path)) @@ -635,30 +634,27 @@ pub async fn read_files( return Ok(Vec::new()); } let cancel = CancellationToken::new(); - let lookup = checkout - .lookup_batch_no_follow(&namespaces, WorkCounters::UNBOUNDED, &cancel) + let reader = checkout + .pinned_reader() + .map_err(EngineError::fs("open pinned reader"))?; + let descriptions = reader + .describe_files(&namespaces, WorkCounters::UNBOUNDED, &cancel) .await - .map_err(EngineError::fs("batch lookup"))? + .map_err(EngineError::fs("describe files"))? .value; let limits = checkout.volume_config().limits; let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); let mut contents = vec![None; paths.len()]; let mut requests = Vec::new(); let mut destinations = Vec::new(); - for (index, entry) in lookup.entries.into_iter().enumerate() { - let Some(record) = entry - .record - .filter(|record| record.kind == FileKind::Regular) - else { + for (index, description) in descriptions.into_iter().enumerate() { + let Some(description) = description else { continue; }; - let length = match record.payload { - acyclic_fs::kernel::FilePayload::InlineRegular(inline) => { - inline.as_bytes().len() as u64 - } - acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, - _ => unreachable!("regular records have regular payloads"), - }; + if description.kind != FileKind::Regular { + continue; + } + let length = description.logical_bytes; let display_path = paths .get(index) .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))?; @@ -675,8 +671,11 @@ pub async fn read_files( let mut offset = 0; while offset < length { let take = chunk.min(length - offset); - requests.push(FileRecordRangeReadRequest { - record, + let path = namespaces + .get(index) + .ok_or_else(|| EngineError::Fs("description returned too many entries".into()))?; + requests.push(FileRangeReadRequest { + path: path.clone(), range: ByteRange { offset, length: take, @@ -689,11 +688,8 @@ pub async fn read_files( if requests.is_empty() { return Ok(contents); } - let reader = checkout - .pinned_reader() - .map_err(EngineError::fs("open pinned reader"))?; let reads = reader - .read_file_record_ranges( + .read_file_ranges( &requests, FILE_READ_CONCURRENCY, WorkCounters::UNBOUNDED, @@ -975,17 +971,21 @@ async fn copy_node( let reader = src .pinned_reader() .map_err(EngineError::fs("open pinned reader"))?; - let metadata = reader - .read_record_metadata_batch( - &nodes.iter().map(|(_, record)| *record).collect::>(), - FILE_READ_CONCURRENCY, - WorkCounters::UNBOUNDED, - cancel, - ) + let node_paths = nodes + .iter() + .map(|(path, _)| path.clone()) + .collect::>(); + let descriptions = reader + .describe_files(&node_paths, WorkCounters::UNBOUNDED, cancel) .await .map_err(EngineError::fs("batch subtree metadata"))? - .value; - let regular = read_regular_frontier(&reader, &nodes, cancel).await?; + .value + .into_iter() + .map(|description| { + description.ok_or_else(|| EngineError::Fs("subtree node is absent".into())) + }) + .collect::>>()?; + let regular = read_regular_frontier(&reader, &node_paths, &descriptions, cancel).await?; let directories = nodes .iter() .filter(|(_, record)| record.kind == FileKind::Directory) @@ -993,7 +993,10 @@ async fn copy_node( .collect::>(); frontier = list_child_paths(src, &directories, cancel).await?; let mut mutations = Vec::with_capacity(nodes.len()); - for (index, ((path, record), metadata)) in nodes.into_iter().zip(metadata).enumerate() { + for (index, ((path, record), description)) in + nodes.into_iter().zip(descriptions).enumerate() + { + let metadata = description.metadata; match record.kind { FileKind::Regular => { let bytes = regular.get(index).ok_or_else(|| { @@ -1007,7 +1010,7 @@ async fn copy_node( } FileKind::SymbolicLink => { let target = reader - .read_symbolic_link_record(record, WorkCounters::UNBOUNDED, cancel) + .read_symbolic_link(&path, WorkCounters::UNBOUNDED, cancel) .await .map_err(EngineError::fs("read resolved symlink"))? .value; @@ -1042,33 +1045,33 @@ async fn copy_node( async fn read_regular_frontier( reader: &acyclic_fs::PinnedReader, - nodes: &[(NamespacePath, FileRecord)], + paths: &[NamespacePath], + descriptions: &[acyclic_fs::FileDescription], cancel: &CancellationToken, ) -> Result>> where A: acyclic_fs::AsyncAuthorityStore, O: acyclic_fs::AsyncObjectStore, { - let mut contents = vec![Vec::new(); nodes.len()]; - let mut offsets = vec![0_u64; nodes.len()]; + let mut contents = vec![Vec::new(); paths.len()]; + let mut offsets = vec![0_u64; paths.len()]; loop { let mut destinations = Vec::new(); let mut requests = Vec::new(); - for (index, ((_, record), offset)) in nodes.iter().zip(&mut offsets).enumerate() { - let length = match record.payload { - acyclic_fs::kernel::FilePayload::InlineRegular(inline) => { - inline.as_bytes().len() as u64 - } - acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, - _ => continue, - }; + for (index, ((path, description), offset)) in + paths.iter().zip(descriptions).zip(&mut offsets).enumerate() + { + if description.kind != FileKind::Regular { + continue; + } + let length = description.logical_bytes; if *offset >= length { continue; } let take = TRANSFER_BYTES.min(length - *offset); destinations.push(index); - requests.push(FileRecordRangeReadRequest { - record: *record, + requests.push(FileRangeReadRequest { + path: path.clone(), range: ByteRange { offset: *offset, length: take, @@ -1080,7 +1083,7 @@ where break; } let read = reader - .read_file_record_ranges( + .read_file_ranges( &requests, FILE_READ_CONCURRENCY, WorkCounters::UNBOUNDED, From 71328e2e9c90f93ccc842607827434f0c5c85b2d Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 14:51:22 +0100 Subject: [PATCH 084/106] Share rewind operation identity Signed-off-by: Var1377 --- crates/acyclic/src/pipeline.rs | 5 +---- crates/acyclic/src/rewind.rs | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 403db82..607d82f 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -2161,16 +2161,13 @@ impl Pipeline { .await .map_err(EngineError::fs("rewind generation"))?; prepared.mark_restoring_head()?; - let key_bytes: [u8; 16] = generation.digest().as_bytes()[..16] - .try_into() - .map_err(|_| EngineError::Store("invalid generation digest".into()))?; match self .store .workspace .restore_generation( &target, current.id(), - acyclic_fs::IdempotencyKey::from_bytes(key_bytes), + crate::rewind::publication_key(generation)?, ) .await .map_err(EngineError::fs("restore workspace generation"))? diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 4d71db3..9eda577 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -8,7 +8,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use acyclic_fs::{ durable_rename, exchange_native_entries, materialize_checkout, materialize_checkout_host_path, - publish_native_exchange, MaterializeError, MaterializeOptions, RenameMode, + publish_native_exchange, IdempotencyKey, MaterializeError, MaterializeOptions, RenameMode, }; use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; use serde::{Deserialize, Serialize}; @@ -22,6 +22,13 @@ const MAXIMUM_EXTENT_SPANS: u32 = 65_536; const TRANSFER_BYTES: u64 = 8 * 1024 * 1024; static PARK_SEQUENCE: AtomicU64 = AtomicU64::new(0); +pub(crate) fn publication_key(generation: GenerationId) -> Result { + let bytes = generation.digest().as_bytes()[..16] + .try_into() + .map_err(|_| EngineError::Store("invalid generation digest".into()))?; + Ok(IdempotencyKey::from_bytes(bytes)) +} + /// What a completed rewind reports back. #[derive(Clone, Debug)] pub struct RewindOutcome { @@ -393,6 +400,7 @@ pub async fn recover_workspace(store: &mut Store, recovered: RecoveredSwap) -> R &store.paths.rewind_journal(), &journal.repo_root, &journal.tmp, + publication_key(recovered.target)?, journal.carried, ) .map_err(|error| EngineError::Restore(format!("recover publish tree: {error}")))?; @@ -599,8 +607,14 @@ impl PreparedRewind<'_> { // the SDK owns the durable carry/exchange journal at the same path. std::fs::remove_file(&self.journal_path)?; let locator = publish_locator(repo, &self.journal_path)?; - let exchange = publish_native_exchange(&self.journal_path, repo, &self.tmp, self.carried) - .map_err(|error| EngineError::Restore(format!("publish tree: {error}")))?; + let exchange = publish_native_exchange( + &self.journal_path, + repo, + &self.tmp, + publication_key(self.target)?, + self.carried, + ) + .map_err(|error| EngineError::Restore(format!("publish tree: {error}")))?; if !exchange.published { return Err(EngineError::Restore( "native exchange recovered without publishing the target tree".into(), From 64e9f15f5390d78b1cbb0c767d622a49afd7379a Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 14:58:03 +0100 Subject: [PATCH 085/106] Reuse pinned file resolution Signed-off-by: Var1377 --- crates/acyclic/src/merge.rs | 136 ++++++++++++++---------------------- 1 file changed, 52 insertions(+), 84 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 65894ba..b837c45 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -22,8 +22,8 @@ use acyclic_fs::text_merge::{ merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, }; use acyclic_fs::{ - AuthoredMutation, ByteRange, CancellationToken, DirectoryPageRequest, FileRangeReadRequest, - GenerationId, WorkCounters, + AuthoredMutation, ByteRange, CancellationToken, DirectoryPageRequest, GenerationId, + WorkCounters, }; use bytes::Bytes; @@ -637,20 +637,19 @@ pub async fn read_files( let reader = checkout .pinned_reader() .map_err(EngineError::fs("open pinned reader"))?; - let descriptions = reader - .describe_files(&namespaces, WorkCounters::UNBOUNDED, &cancel) + let files = reader + .resolve_files(&namespaces, WorkCounters::UNBOUNDED, &cancel) .await - .map_err(EngineError::fs("describe files"))? + .map_err(EngineError::fs("resolve files"))? .value; let limits = checkout.volume_config().limits; let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); let mut contents = vec![None; paths.len()]; - let mut requests = Vec::new(); - let mut destinations = Vec::new(); - for (index, description) in descriptions.into_iter().enumerate() { - let Some(description) = description else { + for (index, file) in files.into_iter().enumerate() { + let Some(file) = file else { continue; }; + let description = file.description(); if description.kind != FileKind::Regular { continue; } @@ -664,46 +663,28 @@ pub async fn read_files( display_path.display() )) })?; - let content = contents - .get_mut(index) - .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))?; - *content = Some(Vec::with_capacity(capacity)); - let mut offset = 0; + let mut content = Vec::with_capacity(capacity); + let mut offset = 0_u64; while offset < length { let take = chunk.min(length - offset); - let path = namespaces - .get(index) - .ok_or_else(|| EngineError::Fs("description returned too many entries".into()))?; - requests.push(FileRangeReadRequest { - path: path.clone(), - range: ByteRange { - offset, - length: take, - }, - }); - destinations.push(index); + let read = file + .read_range( + ByteRange { + offset, + length: take, + }, + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(EngineError::fs("read resolved file range"))?; + content.extend_from_slice(&read.value.bytes); offset += take; } - } - if requests.is_empty() { - return Ok(contents); - } - let reads = reader - .read_file_ranges( - &requests, - FILE_READ_CONCURRENCY, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("batch read file ranges"))? - .value; - for (destination, read) in destinations.into_iter().zip(reads) { - contents - .get_mut(destination) - .and_then(Option::as_mut) - .ok_or_else(|| EngineError::Fs("batch read returned an invalid destination".into()))? - .extend_from_slice(&read.bytes); + *contents + .get_mut(index) + .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))? = + Some(content); } Ok(contents) } @@ -975,17 +956,15 @@ async fn copy_node( .iter() .map(|(path, _)| path.clone()) .collect::>(); - let descriptions = reader - .describe_files(&node_paths, WorkCounters::UNBOUNDED, cancel) + let files = reader + .resolve_files(&node_paths, WorkCounters::UNBOUNDED, cancel) .await .map_err(EngineError::fs("batch subtree metadata"))? .value .into_iter() - .map(|description| { - description.ok_or_else(|| EngineError::Fs("subtree node is absent".into())) - }) + .map(|file| file.ok_or_else(|| EngineError::Fs("subtree node is absent".into()))) .collect::>>()?; - let regular = read_regular_frontier(&reader, &node_paths, &descriptions, cancel).await?; + let regular = read_regular_frontier(&files, cancel).await?; let directories = nodes .iter() .filter(|(_, record)| record.kind == FileKind::Directory) @@ -993,10 +972,8 @@ async fn copy_node( .collect::>(); frontier = list_child_paths(src, &directories, cancel).await?; let mut mutations = Vec::with_capacity(nodes.len()); - for (index, ((path, record), description)) in - nodes.into_iter().zip(descriptions).enumerate() - { - let metadata = description.metadata; + for (index, ((path, record), file)) in nodes.into_iter().zip(files).enumerate() { + let metadata = file.description().metadata; match record.kind { FileKind::Regular => { let bytes = regular.get(index).ok_or_else(|| { @@ -1009,8 +986,8 @@ async fn copy_node( }); } FileKind::SymbolicLink => { - let target = reader - .read_symbolic_link(&path, WorkCounters::UNBOUNDED, cancel) + let target = file + .read_symbolic_link(WorkCounters::UNBOUNDED, cancel) .await .map_err(EngineError::fs("read resolved symlink"))? .value; @@ -1044,23 +1021,20 @@ async fn copy_node( } async fn read_regular_frontier( - reader: &acyclic_fs::PinnedReader, - paths: &[NamespacePath], - descriptions: &[acyclic_fs::FileDescription], + files: &[acyclic_fs::ResolvedFile<'_, A, O>], cancel: &CancellationToken, ) -> Result>> where A: acyclic_fs::AsyncAuthorityStore, O: acyclic_fs::AsyncObjectStore, { - let mut contents = vec![Vec::new(); paths.len()]; - let mut offsets = vec![0_u64; paths.len()]; + let mut contents = vec![Vec::new(); files.len()]; + let mut offsets = vec![0_u64; files.len()]; loop { let mut destinations = Vec::new(); - let mut requests = Vec::new(); - for (index, ((path, description), offset)) in - paths.iter().zip(descriptions).zip(&mut offsets).enumerate() - { + let mut pending = Vec::new(); + for (index, (file, offset)) in files.iter().zip(&mut offsets).enumerate() { + let description = file.description(); if description.kind != FileKind::Regular { continue; } @@ -1069,34 +1043,28 @@ where continue; } let take = TRANSFER_BYTES.min(length - *offset); - destinations.push(index); - requests.push(FileRangeReadRequest { - path: path.clone(), - range: ByteRange { + pending.push(( + file, + ByteRange { offset: *offset, length: take, }, - }); + )); + destinations.push(index); *offset += take; } - if requests.is_empty() { + if pending.is_empty() { break; } - let read = reader - .read_file_ranges( - &requests, - FILE_READ_CONCURRENCY, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("batch read subtree ranges"))? - .value; - for (index, chunk) in destinations.into_iter().zip(read) { + for (index, (file, range)) in destinations.into_iter().zip(pending) { + let chunk = file + .read_range(range, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("read resolved subtree range"))?; contents .get_mut(index) .ok_or_else(|| EngineError::Fs("batch read returned an invalid index".into()))? - .extend_from_slice(&chunk.bytes); + .extend_from_slice(&chunk.value.bytes); } } Ok(contents) From 280c2aac15cbd562167314fd2276c7e7a88ea047 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 15:02:52 +0100 Subject: [PATCH 086/106] Overlap resolved subtree reads Signed-off-by: Var1377 --- crates/acyclic/src/merge.rs | 80 +++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 29 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index b837c45..547ae0f 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -23,7 +23,7 @@ use acyclic_fs::text_merge::{ }; use acyclic_fs::{ AuthoredMutation, ByteRange, CancellationToken, DirectoryPageRequest, GenerationId, - WorkCounters, + ResolvedFileRangeReadRequest, WorkCounters, }; use bytes::Bytes; @@ -645,7 +645,9 @@ pub async fn read_files( let limits = checkout.volume_config().limits; let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); let mut contents = vec![None; paths.len()]; - for (index, file) in files.into_iter().enumerate() { + let mut pending = Vec::new(); + let mut destinations = Vec::new(); + for (index, file) in files.iter().enumerate() { let Some(file) = file else { continue; }; @@ -663,28 +665,41 @@ pub async fn read_files( display_path.display() )) })?; - let mut content = Vec::with_capacity(capacity); + *contents + .get_mut(index) + .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))? = + Some(Vec::with_capacity(capacity)); let mut offset = 0_u64; while offset < length { let take = chunk.min(length - offset); - let read = file - .read_range( - ByteRange { - offset, - length: take, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("read resolved file range"))?; - content.extend_from_slice(&read.value.bytes); + pending.push(ResolvedFileRangeReadRequest { + file, + range: ByteRange { + offset, + length: take, + }, + }); + destinations.push(index); offset += take; } - *contents - .get_mut(index) - .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))? = - Some(content); + } + let reads = reader + .read_resolved_ranges( + &pending, + FILE_READ_CONCURRENCY, + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(EngineError::fs("read resolved file ranges"))? + .value; + for (destination, read) in destinations.into_iter().zip(reads) { + contents + .get_mut(destination) + .ok_or_else(|| EngineError::Fs("resolved read has an invalid destination".into()))? + .as_mut() + .ok_or_else(|| EngineError::Fs("resolved read lost its destination".into()))? + .extend_from_slice(&read.bytes); } Ok(contents) } @@ -964,7 +979,7 @@ async fn copy_node( .into_iter() .map(|file| file.ok_or_else(|| EngineError::Fs("subtree node is absent".into()))) .collect::>>()?; - let regular = read_regular_frontier(&files, cancel).await?; + let regular = read_regular_frontier(&reader, &files, cancel).await?; let directories = nodes .iter() .filter(|(_, record)| record.kind == FileKind::Directory) @@ -1021,6 +1036,7 @@ async fn copy_node( } async fn read_regular_frontier( + reader: &acyclic_fs::PinnedReader, files: &[acyclic_fs::ResolvedFile<'_, A, O>], cancel: &CancellationToken, ) -> Result>> @@ -1043,28 +1059,34 @@ where continue; } let take = TRANSFER_BYTES.min(length - *offset); - pending.push(( + pending.push(ResolvedFileRangeReadRequest { file, - ByteRange { + range: ByteRange { offset: *offset, length: take, }, - )); + }); destinations.push(index); *offset += take; } if pending.is_empty() { break; } - for (index, (file, range)) in destinations.into_iter().zip(pending) { - let chunk = file - .read_range(range, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("read resolved subtree range"))?; + let chunks = reader + .read_resolved_ranges( + &pending, + FILE_READ_CONCURRENCY, + WorkCounters::UNBOUNDED, + cancel, + ) + .await + .map_err(EngineError::fs("batch read resolved subtree ranges"))? + .value; + for (index, chunk) in destinations.into_iter().zip(chunks) { contents .get_mut(index) .ok_or_else(|| EngineError::Fs("batch read returned an invalid index".into()))? - .extend_from_slice(&chunk.value.bytes); + .extend_from_slice(&chunk.bytes); } } Ok(contents) From 4f29567b94252c763cc38232ec0e0559841cc917 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 15:12:42 +0100 Subject: [PATCH 087/106] Reuse resolved subtree traversal Signed-off-by: Var1377 --- crates/acyclic/src/merge.rs | 139 ++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 78 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 547ae0f..ad63205 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -22,8 +22,8 @@ use acyclic_fs::text_merge::{ merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, }; use acyclic_fs::{ - AuthoredMutation, ByteRange, CancellationToken, DirectoryPageRequest, GenerationId, - ResolvedFileRangeReadRequest, WorkCounters, + AuthoredMutation, ByteRange, CancellationToken, GenerationId, ResolvedFileRangeReadRequest, + WorkCounters, }; use bytes::Bytes; @@ -831,58 +831,43 @@ fn child_path( .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}"))) } -async fn list_child_paths( - checkout: &LocalCheckout, - directories: &[NamespacePath], +async fn resolved_child_paths( + reader: &acyclic_fs::PinnedReader, + directory: &NamespacePath, + limits: acyclic_fs::model::VolumeLimits, cancel: &CancellationToken, -) -> Result> { - if directories.is_empty() { - return Ok(Vec::new()); - } - let limits = checkout.volume_config().limits; - let reader = checkout - .pinned_reader() - .map_err(EngineError::fs("open directory reader"))?; - let mut pending = directories - .iter() - .cloned() - .map(|path| DirectoryPageRequest { - path, - after: None, - maximum_entries: PAGE_ENTRIES, - }) - .collect::>(); +) -> Result> +where + A: acyclic_fs::AsyncAuthorityStore, + O: acyclic_fs::AsyncObjectStore, +{ let mut children = Vec::new(); - while !pending.is_empty() { - let pages = reader - .list_directory_pages( - &pending, - FILE_READ_CONCURRENCY, + let mut after = None; + loop { + let page = reader + .resolve_directory_page( + directory, + after.as_ref(), + PAGE_ENTRIES, WorkCounters::UNBOUNDED, cancel, ) .await - .map_err(EngineError::fs("batch list directories"))? + .map_err(EngineError::fs("resolve subtree page"))? .value; - let mut next = Vec::new(); - for (request, page) in pending.into_iter().zip(pages) { - for entry in &page.entries { - children.push(child_path(&request.path, &entry.name, limits)?); - } - if page.has_more { - let after = page - .entries - .last() - .ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))? - .name - .clone(); - next.push(DirectoryPageRequest { - after: Some(after), - ..request - }); - } + for entry in &page.entries { + children.push(child_path(directory, &entry.name, limits)?); + } + if !page.has_more { + break; } - pending = next; + after = Some( + page.entries + .last() + .ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))? + .name + .clone(), + ); } Ok(children) } @@ -912,12 +897,18 @@ async fn remove_subtree( if nodes.is_empty() { break; } - let directories = nodes + let reader = dst + .pinned_reader() + .map_err(EngineError::fs("open subtree reader"))?; + let limits = dst.volume_config().limits; + frontier = Vec::new(); + for directory in nodes .iter() .filter(|(_, record)| record.kind == FileKind::Directory) - .map(|(path, _)| path.clone()) - .collect::>(); - frontier = list_child_paths(dst, &directories, cancel).await?; + .map(|(path, _)| path) + { + frontier.extend(resolved_child_paths(&reader, directory, limits, cancel).await?); + } levels.push(nodes); } let maximum = usize::try_from(dst.volume_config().limits.maximum_mutations_per_batch) @@ -951,45 +942,37 @@ async fn copy_node( ) -> Result<()> { let mut frontier = vec![namespace.clone()]; while !frontier.is_empty() { - let lookup = src - .lookup_batch_no_follow(&frontier, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("batch subtree lookup"))? - .value; - let nodes = frontier - .into_iter() - .zip(lookup.entries) - .filter_map(|(path, entry)| entry.record.map(|record| (path, record))) - .collect::>(); - if nodes.is_empty() { - break; - } let reader = src .pinned_reader() .map_err(EngineError::fs("open pinned reader"))?; - let node_paths = nodes - .iter() - .map(|(path, _)| path.clone()) - .collect::>(); let files = reader - .resolve_files(&node_paths, WorkCounters::UNBOUNDED, cancel) + .resolve_files(&frontier, WorkCounters::UNBOUNDED, cancel) .await .map_err(EngineError::fs("batch subtree metadata"))? .value + .into_iter(); + let nodes = frontier .into_iter() - .map(|file| file.ok_or_else(|| EngineError::Fs("subtree node is absent".into()))) - .collect::>>()?; + .zip(files) + .filter_map(|(path, file)| file.map(|file| (path, file))) + .collect::>(); + if nodes.is_empty() { + break; + } + let files = nodes.iter().map(|(_, file)| file).collect::>(); let regular = read_regular_frontier(&reader, &files, cancel).await?; - let directories = nodes + let limits = src.volume_config().limits; + frontier = Vec::new(); + for (directory, _) in nodes .iter() - .filter(|(_, record)| record.kind == FileKind::Directory) - .map(|(path, _)| path.clone()) - .collect::>(); - frontier = list_child_paths(src, &directories, cancel).await?; + .filter(|(_, file)| file.description().kind == FileKind::Directory) + { + frontier.extend(resolved_child_paths(&reader, directory, limits, cancel).await?); + } let mut mutations = Vec::with_capacity(nodes.len()); - for (index, ((path, record), file)) in nodes.into_iter().zip(files).enumerate() { + for (index, (path, file)) in nodes.into_iter().enumerate() { let metadata = file.description().metadata; - match record.kind { + match file.description().kind { FileKind::Regular => { let bytes = regular.get(index).ok_or_else(|| { EngineError::Fs("subtree read omitted a regular file".into()) @@ -1037,7 +1020,7 @@ async fn copy_node( async fn read_regular_frontier( reader: &acyclic_fs::PinnedReader, - files: &[acyclic_fs::ResolvedFile<'_, A, O>], + files: &[&acyclic_fs::ResolvedFile<'_, A, O>], cancel: &CancellationToken, ) -> Result>> where From 6218a1c272e378b98482db746ec4acbc3588a453 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 16:31:21 +0100 Subject: [PATCH 088/106] Track owned resolved file handles Signed-off-by: Var1377 --- crates/acyclic/src/merge.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index ad63205..fb40b6d 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -15,6 +15,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use acyclic_fs::kernel::{FileKind, FileMetadata, FileRecord, MetadataField, NamespacePath}; +use acyclic_fs::model::GenerationSelector; pub use acyclic_fs::text_merge::{ conflict_hunks, has_conflict_markers, ByteConflictKind as ConflictKind, }; @@ -29,7 +30,7 @@ use bytes::Bytes; use crate::diff; use crate::rewind::validate_relative; -use crate::store::{LocalCheckout, Store}; +use crate::store::{read_only, LocalCheckout, LocalWorkspace, Store}; use crate::{EngineError, Result}; /// Default `[merge] max_file_bytes`. @@ -734,7 +735,7 @@ pub async fn apply_entries( match entry { Entry::Regular { bytes, mode } => { source = None; - remove_subtree(dst, namespace, &cancel).await?; + remove_subtree(&store.workspace, dst, namespace, &cancel).await?; let metadata = FileMetadata { posix_mode: mode.map_or(MetadataField::Unavailable, MetadataField::Value), ..FileMetadata::default() @@ -765,7 +766,7 @@ pub async fn apply_entries( .as_mut() .ok_or_else(|| EngineError::Fs("source checkout missing".into()))? .1; - remove_subtree(dst, namespace, &cancel).await?; + remove_subtree(&store.workspace, dst, namespace, &cancel).await?; copy_node(src, dst, namespace, &cancel).await?; } } @@ -877,6 +878,7 @@ where /// futures are large, and nesting them on the pipeline thread's stack /// overflows it within a few levels. async fn remove_subtree( + workspace: &LocalWorkspace, dst: &mut LocalCheckout, namespace: &NamespacePath, cancel: &CancellationToken, @@ -897,7 +899,11 @@ async fn remove_subtree( if nodes.is_empty() { break; } - let reader = dst + let pinned = workspace + .checkout(GenerationSelector::Exact(dst.generation_id()), read_only()) + .await + .map_err(EngineError::fs("open subtree checkout"))?; + let reader = pinned .pinned_reader() .map_err(EngineError::fs("open subtree reader"))?; let limits = dst.volume_config().limits; @@ -1020,7 +1026,7 @@ async fn copy_node( async fn read_regular_frontier( reader: &acyclic_fs::PinnedReader, - files: &[&acyclic_fs::ResolvedFile<'_, A, O>], + files: &[&acyclic_fs::ResolvedFile], cancel: &CancellationToken, ) -> Result>> where From 8e37518f05c901bd6cf5b821558af77b7db9490a Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 16:34:55 +0100 Subject: [PATCH 089/106] Consume lazy generation readers Signed-off-by: Var1377 --- crates/acyclic/src/merge.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index fb40b6d..e9e6133 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -15,7 +15,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use acyclic_fs::kernel::{FileKind, FileMetadata, FileRecord, MetadataField, NamespacePath}; -use acyclic_fs::model::GenerationSelector; pub use acyclic_fs::text_merge::{ conflict_hunks, has_conflict_markers, ByteConflictKind as ConflictKind, }; @@ -30,7 +29,7 @@ use bytes::Bytes; use crate::diff; use crate::rewind::validate_relative; -use crate::store::{read_only, LocalCheckout, LocalWorkspace, Store}; +use crate::store::{LocalCheckout, Store}; use crate::{EngineError, Result}; /// Default `[merge] max_file_bytes`. @@ -735,7 +734,7 @@ pub async fn apply_entries( match entry { Entry::Regular { bytes, mode } => { source = None; - remove_subtree(&store.workspace, dst, namespace, &cancel).await?; + remove_subtree(store, dst, namespace, &cancel).await?; let metadata = FileMetadata { posix_mode: mode.map_or(MetadataField::Unavailable, MetadataField::Value), ..FileMetadata::default() @@ -766,7 +765,7 @@ pub async fn apply_entries( .as_mut() .ok_or_else(|| EngineError::Fs("source checkout missing".into()))? .1; - remove_subtree(&store.workspace, dst, namespace, &cancel).await?; + remove_subtree(store, dst, namespace, &cancel).await?; copy_node(src, dst, namespace, &cancel).await?; } } @@ -878,7 +877,7 @@ where /// futures are large, and nesting them on the pipeline thread's stack /// overflows it within a few levels. async fn remove_subtree( - workspace: &LocalWorkspace, + store: &Store, dst: &mut LocalCheckout, namespace: &NamespacePath, cancel: &CancellationToken, @@ -899,12 +898,11 @@ async fn remove_subtree( if nodes.is_empty() { break; } - let pinned = workspace - .checkout(GenerationSelector::Exact(dst.generation_id()), read_only()) + let reader = store + .generation(dst.generation_id()) + .await? + .reader() .await - .map_err(EngineError::fs("open subtree checkout"))?; - let reader = pinned - .pinned_reader() .map_err(EngineError::fs("open subtree reader"))?; let limits = dst.volume_config().limits; frontier = Vec::new(); From a9808cc0f1451521d6e690f4a4bccad7435f9fd6 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 16:52:25 +0100 Subject: [PATCH 090/106] Traverse subtree records once Signed-off-by: Fixture --- crates/acyclic/src/merge.rs | 93 +++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 29 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index e9e6133..6ea5a25 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -734,7 +734,7 @@ pub async fn apply_entries( match entry { Entry::Regular { bytes, mode } => { source = None; - remove_subtree(store, dst, namespace, &cancel).await?; + remove_subtree(dst, namespace, &cancel).await?; let metadata = FileMetadata { posix_mode: mode.map_or(MetadataField::Unavailable, MetadataField::Value), ..FileMetadata::default() @@ -765,7 +765,7 @@ pub async fn apply_entries( .as_mut() .ok_or_else(|| EngineError::Fs("source checkout missing".into()))? .1; - remove_subtree(store, dst, namespace, &cancel).await?; + remove_subtree(dst, namespace, &cancel).await?; copy_node(src, dst, namespace, &cancel).await?; } } @@ -872,48 +872,83 @@ where Ok(children) } +async fn resolved_children( + reader: &acyclic_fs::PinnedReader, + directory: &NamespacePath, + limits: acyclic_fs::model::VolumeLimits, + cancel: &CancellationToken, +) -> Result)>> +where + A: acyclic_fs::AsyncAuthorityStore, + O: acyclic_fs::AsyncObjectStore, +{ + let mut children = Vec::new(); + let mut after = None; + loop { + let page = reader + .resolve_directory_page( + directory, + after.as_ref(), + PAGE_ENTRIES, + WorkCounters::UNBOUNDED, + cancel, + ) + .await + .map_err(EngineError::fs("resolve subtree page"))? + .value; + let next = page.entries.last().map(|entry| entry.name.clone()); + for entry in page.entries { + children.push((child_path(directory, &entry.name, limits)?, entry.file)); + } + if !page.has_more { + break; + } + after = Some( + next.ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))?, + ); + } + Ok(children) +} + /// Removes `namespace` from `dst`, recursively for directories; absent is /// fine. Iterative (post-order over an explicit stack): the fs facade's /// futures are large, and nesting them on the pipeline thread's stack /// overflows it within a few levels. async fn remove_subtree( - store: &Store, dst: &mut LocalCheckout, namespace: &NamespacePath, cancel: &CancellationToken, ) -> Result<()> { let mut frontier = vec![namespace.clone()]; + let reader = dst.snapshot_reader(); + let roots = reader + .resolve_files(&frontier, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("resolve subtree root"))? + .value; + let mut nodes = frontier + .drain(..) + .zip(roots) + .filter_map(|(path, file)| file.map(|file| (path, file))) + .collect::>(); let mut levels = Vec::new(); - while !frontier.is_empty() { - let lookup = dst - .lookup_batch_no_follow(&frontier, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("batch subtree removal lookup"))? - .value; - let nodes = frontier - .into_iter() - .zip(lookup.entries) - .filter_map(|(path, entry)| entry.record.map(|record| (path, record))) - .collect::>(); - if nodes.is_empty() { - break; - } - let reader = store - .generation(dst.generation_id()) - .await? - .reader() - .await - .map_err(EngineError::fs("open subtree reader"))?; + while !nodes.is_empty() { let limits = dst.volume_config().limits; - frontier = Vec::new(); + let mut next = Vec::new(); for directory in nodes .iter() - .filter(|(_, record)| record.kind == FileKind::Directory) + .filter(|(_, file)| file.description().kind == FileKind::Directory) .map(|(path, _)| path) { - frontier.extend(resolved_child_paths(&reader, directory, limits, cancel).await?); + next.extend(resolved_children(&reader, directory, limits, cancel).await?); } - levels.push(nodes); + levels.push( + nodes + .into_iter() + .map(|(path, file)| (path, file.file_id())) + .collect::>(), + ); + nodes = next; } let maximum = usize::try_from(dst.volume_config().limits.maximum_mutations_per_batch) .unwrap_or(usize::MAX) @@ -921,9 +956,9 @@ async fn remove_subtree( for level in levels.into_iter().rev() { let removals = level .into_iter() - .map(|(path, record)| AuthoredMutation::Remove { + .map(|(path, file_id)| AuthoredMutation::Remove { path, - expected_file_id: Some(record.file_id), + expected_file_id: Some(file_id), }) .collect::>(); for chunk in removals.chunks(maximum) { From d0ad607a2d51751d896e956eec7f61a8112a5b5c Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 17:02:54 +0100 Subject: [PATCH 091/106] Qualify against the active SDK branch Signed-off-by: Fixture --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 101c2ee..cc8261c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: acyclic-labs/sdk - ref: codex/consumer-driven-lab + ref: codex/generation-reader path: sdk persist-credentials: false - name: Install FUSE-T on macOS @@ -48,7 +48,7 @@ jobs: ~/.cargo/registry ~/.cargo/git sdk/target-* - key: qualify-${{ matrix.os }}-${{ hashFiles('sdk/Cargo.lock', 'worktrees/graphcoder/plugin-sdk-lab/Cargo.lock') }} + key: qualify-${{ matrix.os }}-${{ hashFiles('sdk/Cargo.lock', 'sdk/rust/**/*.rs', 'worktrees/graphcoder/plugin-sdk-lab/Cargo.lock', 'worktrees/graphcoder/plugin-sdk-lab/crates/**/*.rs') }} restore-keys: | qualify-${{ matrix.os }}- - name: Run the shared bounded gate From 1ef0f58903c1adbde72bce577cddddd851b447c4 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 17:31:26 +0100 Subject: [PATCH 092/106] Make qualification prerequisites exact Signed-off-by: Var1377 --- .github/workflows/ci.yml | 4 ++++ crates/acyclic/src/store.rs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc8261c..3e5a6b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,10 @@ jobs: https://github.com/macos-fuse-t/fuse-t/releases/download/1.2.7/fuse-t-macos-installer-1.2.7.pkg \ --output /tmp/fuse-t.pkg sudo installer -pkg /tmp/fuse-t.pkg -target / + - name: Install Bun + uses: oven-sh/setup-bun@735343b667d3e6f658f44d0eca948eb6282f2b76 # v2.0.2 + with: + bun-version: 1.3.14 - name: Cache immutable dependencies and build outputs uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index 20c98f4..8756b4e 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -3,6 +3,7 @@ //! Everything lives OUTSIDE the working tree (capture snapshots the whole //! tree and fail-closes on sockets). The repo carries only `.acyclic/config.toml`. +#[cfg(target_os = "windows")] use std::io::Write; use std::path::{Path, PathBuf}; @@ -465,6 +466,7 @@ fn atomic_write_json(path: &Path, value: &T) -> Result<()> { Ok(()) } +#[cfg(target_os = "windows")] pub(crate) fn durable_replace(path: &Path, bytes: &[u8]) -> Result<()> { let tmp = path.with_extension("bin.tmp"); let mut file = std::fs::OpenOptions::new() From 537fdc39f88512e0409b1bd4ef629a81c965b335 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 17:38:10 +0100 Subject: [PATCH 093/106] Use platform thread stacks Signed-off-by: Var1377 --- crates/acyclic/src/pipeline.rs | 5 ----- crates/acyclic/src/speculate.rs | 3 --- 2 files changed, 8 deletions(-) diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 607d82f..97866e4 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -708,11 +708,6 @@ pub fn spawn( let (sender, receiver) = mpsc::channel(1024); let thread = std::thread::Builder::new() .name(format!("{}-pipeline", crate::product::NAME)) - // The fs facade's futures are large and a few of them nest per - // request (a subtree copy, a restore); the 2 MiB default is tight - // in debug builds. Virtual reservation only: untouched pages cost - // nothing. - .stack_size(32 * 1024 * 1024) .spawn(move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_time() diff --git a/crates/acyclic/src/speculate.rs b/crates/acyclic/src/speculate.rs index 4c54716..e64b640 100644 --- a/crates/acyclic/src/speculate.rs +++ b/crates/acyclic/src/speculate.rs @@ -300,9 +300,6 @@ pub fn spawn( let thread_config = config.clone(); let thread = std::thread::Builder::new() .name(format!("{}-speculate", acyclic::product::NAME)) - // The fs futures a diff pulls in are large; match the pipeline's - // headroom rather than the 2 MiB default. - .stack_size(32 * 1024 * 1024) .spawn(move || { // `enable_all`, not just timers: a model run is a child // process, which needs the IO and signal drivers. From ca1746fe4f5f5987bd9091f7d7812591d229860e Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 17:46:17 +0100 Subject: [PATCH 094/106] Use generation materialization facade Signed-off-by: Var1377 --- crates/acyclic/src/rewind.rs | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 9eda577..00a75e1 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -7,7 +7,7 @@ use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use acyclic_fs::{ - durable_rename, exchange_native_entries, materialize_checkout, materialize_checkout_host_path, + durable_rename, exchange_native_entries, materialize_checkout_host_path, publish_native_exchange, IdempotencyKey, MaterializeError, MaterializeOptions, RenameMode, }; use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; @@ -550,25 +550,25 @@ pub(crate) async fn prepare<'a>( carried: Vec::new(), }, )?; - let mut checkout = store.checkout_exact(target).await?; + let generation = store.generation(target).await?; let cancel = CancellationToken::new(); - materialize_checkout( - &mut checkout, - &MaterializeOptions { - destination: tmp.clone(), - maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, - maximum_extent_spans: MAXIMUM_EXTENT_SPANS, - transfer_bytes: TRANSFER_BYTES, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(|error| { - let _ = std::fs::remove_dir_all(&tmp); - let _ = std::fs::remove_file(&journal_path); - EngineError::Restore(format!("materialize: {error:?}")) - })?; + generation + .materialize( + &MaterializeOptions { + destination: tmp.clone(), + maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, + maximum_extent_spans: MAXIMUM_EXTENT_SPANS, + transfer_bytes: TRANSFER_BYTES, + }, + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(|error| { + let _ = std::fs::remove_dir_all(&tmp); + let _ = std::fs::remove_file(&journal_path); + EngineError::Restore(format!("materialize: {error:?}")) + })?; let carried: Vec = exclusions .host_paths() From e49ffef7f9f4ffd9017a53c0507ac7cf07525ccd Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 17:52:24 +0100 Subject: [PATCH 095/106] Use SDK native exchange journal Signed-off-by: Var1377 --- crates/acyclic/src/rewind.rs | 46 ++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 00a75e1..2acbe76 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -8,7 +8,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use acyclic_fs::{ durable_rename, exchange_native_entries, materialize_checkout_host_path, - publish_native_exchange, IdempotencyKey, MaterializeError, MaterializeOptions, RenameMode, + prepare_native_exchange_with_recovery, publish_native_exchange, recover_native_exchange, + IdempotencyKey, MaterializeError, MaterializeOptions, NativeExchangeJournal, RenameMode, }; use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; use serde::{Deserialize, Serialize}; @@ -46,6 +47,7 @@ pub(crate) struct PreparedRewind<'a> { parent: PathBuf, name: String, journal_path: PathBuf, + staging_journal_path: PathBuf, carried: Vec, trash_ttl_days: u32, } @@ -526,10 +528,12 @@ pub(crate) async fn prepare<'a>( let nonce = std::process::id(); let tmp = parent.join(format!(".{name}.{}-tmp-{nonce}", crate::product::NAME)); let journal_path = store.paths.rewind_journal(); + let staging_journal_path = journal_path.with_extension("staging.json"); // A failed previous exchange may have left a complete tree in scratch. // Resolve its journal before reusing either the temporary name or journal. recover(&journal_path)?; + let _ = std::fs::remove_file(&staging_journal_path); // 1. Materialize the target into an empty sibling directory. A tmp left // by an earlier attempt that failed before the swap is stale by @@ -541,7 +545,7 @@ pub(crate) async fn prepare<'a>( EngineError::Restore(format!("rewind: stage {}: {error}", tmp.display())) })?; write_journal( - &journal_path, + &staging_journal_path, &Journal { target_generation: hex::encode(target.digest().as_bytes()), repo_root: repo.clone(), @@ -566,7 +570,7 @@ pub(crate) async fn prepare<'a>( .await .map_err(|error| { let _ = std::fs::remove_dir_all(&tmp); - let _ = std::fs::remove_file(&journal_path); + let _ = std::fs::remove_file(&staging_journal_path); EngineError::Restore(format!("materialize: {error:?}")) })?; @@ -582,6 +586,7 @@ pub(crate) async fn prepare<'a>( parent: parent.to_path_buf(), name, journal_path, + staging_journal_path, carried, trash_ttl_days, }) @@ -589,23 +594,20 @@ pub(crate) async fn prepare<'a>( impl PreparedRewind<'_> { pub(crate) fn mark_restoring_head(&self) -> Result<()> { - write_journal( + std::fs::remove_file(&self.staging_journal_path)?; + prepare_native_exchange_with_recovery( &self.journal_path, - &Journal { - target_generation: hex::encode(self.target.digest().as_bytes()), - repo_root: self.store.repo_root.clone(), - tmp: self.tmp.clone(), - phase: Phase::RestoringHead, - carried: self.carried.clone(), - }, + &self.store.repo_root, + &self.tmp, + publication_key(self.target)?, + self.carried.clone(), + self.target.digest().as_bytes().to_vec(), ) + .map_err(|error| EngineError::Restore(format!("prepare tree publication: {error}"))) } pub(crate) fn publish(self) -> Result { let repo = &self.store.repo_root; - // The staging-only legacy phase has served its purpose. From this point - // the SDK owns the durable carry/exchange journal at the same path. - std::fs::remove_file(&self.journal_path)?; let locator = publish_locator(repo, &self.journal_path)?; let exchange = publish_native_exchange( &self.journal_path, @@ -664,7 +666,21 @@ pub fn recover(journal_path: &Path) -> Result> { Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(error.into()), }; - let journal: Journal = serde_json::from_str(&text) + if let Ok(journal) = serde_json::from_str::(&text) { + let target = decode_generation(&hex::encode(&journal.recovery))?; + let outcome = recover_native_exchange(journal_path) + .map_err(|error| EngineError::Restore(format!("recover native exchange: {error}")))?; + return Ok(Some(RecoveredSwap { + published: outcome.published, + old_tree: outcome.displaced, + target, + })); + } + recover_legacy(journal_path, &text) +} + +fn recover_legacy(journal_path: &Path, text: &str) -> Result> { + let journal: Journal = serde_json::from_str(text) .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; let target = decode_generation(&journal.target_generation)?; #[cfg(windows)] From 5858b909bf8925db0bcff9ebeccc90526f596485 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 17:53:36 +0100 Subject: [PATCH 096/106] Recover partial rewind staging directly Signed-off-by: Var1377 --- crates/acyclic/src/rewind.rs | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 2acbe76..d1cbd7f 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -661,6 +661,7 @@ fn park_replaced_tree(tmp: &Path, parent: &Path, name: &str) -> Result /// Startup crash recovery. Reads the journal (if any) and finishes or unwinds /// the interrupted rewind so the repo is whole before the pipeline baselines. pub fn recover(journal_path: &Path) -> Result> { + recover_staging(&journal_path.with_extension("staging.json"))?; let text = match std::fs::read_to_string(journal_path) { Ok(text) => text, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), @@ -679,6 +680,24 @@ pub fn recover(journal_path: &Path) -> Result> { recover_legacy(journal_path, &text) } +fn recover_staging(path: &Path) -> Result<()> { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + let journal: Journal = serde_json::from_str(&text) + .map_err(|error| EngineError::Restore(format!("rewind staging journal: {error}")))?; + if journal.phase != Phase::Materializing { + return Err(EngineError::Restore( + "rewind staging journal has an invalid phase".into(), + )); + } + remove_any(&journal.tmp)?; + std::fs::remove_file(path)?; + Ok(()) +} + fn recover_legacy(journal_path: &Path, text: &str) -> Result> { let journal: Journal = serde_json::from_str(text) .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; @@ -1039,6 +1058,25 @@ mod tests { assert!(!journal_path.exists()); } + #[test] + fn recover_discards_separate_partial_staging() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + std::fs::create_dir(&repo).expect("repo"); + std::fs::write(repo.join("keep.txt"), b"live").expect("seed"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir(&tmp).expect("tmp"); + std::fs::write(tmp.join("partial.txt"), b"half").expect("seed"); + let journal_path = work.path().join("journal.json"); + let staging_path = journal_path.with_extension("staging.json"); + write(&staging_path, &journal(&repo, &tmp, Phase::Materializing)); + + recover(&journal_path).expect("recover"); + assert_eq!(std::fs::read(repo.join("keep.txt")).expect("read"), b"live"); + assert!(!tmp.exists()); + assert!(!staging_path.exists()); + } + #[test] fn recover_with_no_journal_is_a_noop() { let work = tempfile::tempdir().expect("tempdir"); From ff846b454eab1c1f5085e4587bf4a418c9e8e730 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 17:54:55 +0100 Subject: [PATCH 097/106] Isolate legacy rewind recovery Signed-off-by: Var1377 --- crates/acyclic/src/rewind.rs | 65 ++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index d1cbd7f..6aa9c85 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -322,11 +322,11 @@ pub(crate) fn remove_any(path: &Path) -> std::io::Result<()> { /// Crash-recovery journal. Present on disk only while a swap is in flight. #[derive(Debug, Serialize, Deserialize)] -pub struct Journal { +struct LegacyJournal { pub target_generation: String, pub repo_root: PathBuf, pub tmp: PathBuf, - pub phase: Phase, + pub phase: LegacyPhase, /// Excluded paths (repo-relative) moved from the live tree into `tmp` /// before the swap. Absent in journals written before exclusions. #[serde(default)] @@ -358,7 +358,7 @@ fn move_back(from: &Path, into: &Path, relative: &[PathBuf]) -> Result<()> { } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub enum Phase { +enum LegacyPhase { /// Materializing into tmp; repo untouched. Recovery: delete tmp. Materializing, /// The prepared tree is complete and the SDK head is being restored. @@ -392,9 +392,9 @@ pub async fn recover_workspace(store: &mut Store, recovered: RecoveredSwap) -> R Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), Err(error) => return Err(error.into()), }; - let journal: Journal = serde_json::from_str(&text) + let journal: LegacyJournal = serde_json::from_str(&text) .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; - if journal.phase != Phase::RestoringHead { + if journal.phase != LegacyPhase::RestoringHead { return Ok(()); } let locator = publish_locator(&journal.repo_root, &store.paths.rewind_journal())?; @@ -546,11 +546,11 @@ pub(crate) async fn prepare<'a>( })?; write_journal( &staging_journal_path, - &Journal { + &LegacyJournal { target_generation: hex::encode(target.digest().as_bytes()), repo_root: repo.clone(), tmp: tmp.clone(), - phase: Phase::Materializing, + phase: LegacyPhase::Materializing, carried: Vec::new(), }, )?; @@ -686,9 +686,9 @@ fn recover_staging(path: &Path) -> Result<()> { Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), Err(error) => return Err(error.into()), }; - let journal: Journal = serde_json::from_str(&text) + let journal: LegacyJournal = serde_json::from_str(&text) .map_err(|error| EngineError::Restore(format!("rewind staging journal: {error}")))?; - if journal.phase != Phase::Materializing { + if journal.phase != LegacyPhase::Materializing { return Err(EngineError::Restore( "rewind staging journal has an invalid phase".into(), )); @@ -699,7 +699,7 @@ fn recover_staging(path: &Path) -> Result<()> { } fn recover_legacy(journal_path: &Path, text: &str) -> Result> { - let journal: Journal = serde_json::from_str(text) + let journal: LegacyJournal = serde_json::from_str(text) .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; let target = decode_generation(&journal.target_generation)?; #[cfg(windows)] @@ -708,24 +708,24 @@ fn recover_legacy(journal_path: &Path, text: &str) -> Result { + LegacyPhase::Materializing => { // Repo untouched; the partial tmp tree is garbage. let _ = std::fs::remove_dir_all(&journal.tmp); } - Phase::RestoringHead => { + LegacyPhase::RestoringHead => { return Ok(Some(RecoveredSwap { published, old_tree, target, })) } - Phase::Carrying => { + LegacyPhase::Carrying => { // Some excluded paths may already sit in tmp: bring them home, // then drop the unused new tree. move_back(&journal.tmp, &journal.repo_root, &journal.carried)?; let _ = std::fs::remove_dir_all(&journal.tmp); } - Phase::Swapping => { + LegacyPhase::Swapping => { #[cfg(windows)] let scratch = swap_scratch(&journal.repo_root); #[cfg(windows)] @@ -823,7 +823,7 @@ fn path_exists(path: &Path) -> Result { } } -fn write_journal(path: &Path, journal: &Journal) -> Result<()> { +fn write_journal(path: &Path, journal: &LegacyJournal) -> Result<()> { let text = serde_json::to_string(journal) .map_err(|error| EngineError::Restore(format!("encode journal: {error}")))?; let tmp = path.with_extension("tmp"); @@ -1004,8 +1004,8 @@ mod tests { .any(|entry| std::fs::read(entry.path().join(file)).ok().as_deref() == Some(expected)) } - fn journal(repo: &Path, tmp: &Path, phase: Phase) -> Journal { - Journal { + fn journal(repo: &Path, tmp: &Path, phase: LegacyPhase) -> LegacyJournal { + LegacyJournal { target_generation: "00".repeat(32), repo_root: repo.to_path_buf(), tmp: tmp.to_path_buf(), @@ -1014,7 +1014,7 @@ mod tests { } } - fn write(path: &Path, value: &Journal) { + fn write(path: &Path, value: &LegacyJournal) { std::fs::write(path, serde_json::to_string(value).expect("encode")).expect("write"); } @@ -1026,7 +1026,7 @@ mod tests { std::fs::create_dir(&tmp).expect("tmp"); std::fs::write(tmp.join("file.txt"), b"restored").expect("seed"); let journal_path = work.path().join("journal.json"); - write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); + write(&journal_path, &journal(&repo, &tmp, LegacyPhase::Swapping)); let recovered = recover(&journal_path) .expect("recover") @@ -1050,7 +1050,10 @@ mod tests { std::fs::create_dir(&tmp).expect("tmp"); std::fs::write(tmp.join("partial.txt"), b"half").expect("seed"); let journal_path = work.path().join("journal.json"); - write(&journal_path, &journal(&repo, &tmp, Phase::Materializing)); + write( + &journal_path, + &journal(&repo, &tmp, LegacyPhase::Materializing), + ); recover(&journal_path).expect("recover"); assert_eq!(std::fs::read(repo.join("keep.txt")).expect("read"), b"live"); @@ -1069,7 +1072,10 @@ mod tests { std::fs::write(tmp.join("partial.txt"), b"half").expect("seed"); let journal_path = work.path().join("journal.json"); let staging_path = journal_path.with_extension("staging.json"); - write(&staging_path, &journal(&repo, &tmp, Phase::Materializing)); + write( + &staging_path, + &journal(&repo, &tmp, LegacyPhase::Materializing), + ); recover(&journal_path).expect("recover"); assert_eq!(std::fs::read(repo.join("keep.txt")).expect("read"), b"live"); @@ -1096,7 +1102,7 @@ mod tests { std::fs::write(tmp.join(".env"), b"LIVE").expect("moved"); std::fs::write(repo.join("secrets/key.pem"), b"KEY").expect("unmoved"); let journal_path = work.path().join("journal.json"); - let mut entry = journal(&repo, &tmp, Phase::Carrying); + let mut entry = journal(&repo, &tmp, LegacyPhase::Carrying); entry.carried = vec![PathBuf::from(".env"), PathBuf::from("secrets/key.pem")]; write(&journal_path, &entry); @@ -1123,7 +1129,7 @@ mod tests { std::fs::write(tmp.join("file.txt"), b"new").expect("new"); std::fs::write(tmp.join(".env"), b"LIVE").expect("carried"); let journal_path = work.path().join("journal.json"); - let mut entry = journal(&repo, &tmp, Phase::Swapping); + let mut entry = journal(&repo, &tmp, LegacyPhase::Swapping); entry.carried = vec![PathBuf::from(".env")]; write(&journal_path, &entry); @@ -1146,7 +1152,7 @@ mod tests { std::fs::write(repo.join(".env"), b"repo copy").expect("repo data"); std::fs::write(tmp.join(".env"), b"staged copy").expect("staged data"); let journal_path = work.path().join("journal.json"); - let mut entry = journal(&repo, &tmp, Phase::Carrying); + let mut entry = journal(&repo, &tmp, LegacyPhase::Carrying); entry.carried = vec![PathBuf::from(".env")]; write(&journal_path, &entry); @@ -1175,7 +1181,7 @@ mod tests { std::fs::write(repo.join(".env"), b"LIVE").expect("carried"); std::fs::write(tmp.join("file.txt"), b"old").expect("old"); let journal_path = work.path().join("journal.json"); - let mut entry = journal(&repo, &tmp, Phase::Swapping); + let mut entry = journal(&repo, &tmp, LegacyPhase::Swapping); entry.carried = vec![PathBuf::from(".env")]; write(&journal_path, &entry); @@ -1193,7 +1199,7 @@ mod tests { fn journals_written_before_exclusions_still_decode() { let text = r#"{"target_generation":"00","repo_root":"/r","tmp":"/t","phase":"Materializing"}"#; - let journal: Journal = serde_json::from_str(text).expect("decode"); + let journal: LegacyJournal = serde_json::from_str(text).expect("decode"); assert!(journal.carried.is_empty()); } @@ -1217,7 +1223,10 @@ mod tests { let store = work.path().join("custom-store"); std::fs::create_dir(&store).expect("custom store"); let journal_path = store.join("rewind-journal.json"); - write(&journal_path, &journal(&repo, &staged, Phase::Swapping)); + write( + &journal_path, + &journal(&repo, &staged, LegacyPhase::Swapping), + ); let locator = locator_path(&repo).expect("locator path"); write_locator( &locator, @@ -1251,7 +1260,7 @@ mod tests { std::fs::create_dir(&scratch).expect("scratch"); std::fs::write(scratch.join("file.txt"), b"old tree").expect("seed old"); let journal_path = work.path().join("journal.json"); - let mut entry = journal(&repo, &tmp, Phase::Swapping); + let mut entry = journal(&repo, &tmp, LegacyPhase::Swapping); entry.carried = vec![PathBuf::from(".env")]; write(&journal_path, &entry); From 7a8f940aa1a229ffbbea8ff803454ad238e9690b Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 17:57:40 +0100 Subject: [PATCH 098/106] Minimize rewind staging state Signed-off-by: Var1377 --- crates/acyclic/src/rewind.rs | 65 ++++++++++++++---------------------- 1 file changed, 25 insertions(+), 40 deletions(-) diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 6aa9c85..65f4a76 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -333,6 +333,11 @@ struct LegacyJournal { pub carried: Vec, } +#[derive(Serialize, Deserialize)] +struct StagingMarker { + temporary: PathBuf, +} + /// Returns every listed path present in `from` to `into` during legacy /// journal recovery. /// A conflict or I/O failure keeps the journal and both trees for retry. @@ -544,14 +549,10 @@ pub(crate) async fn prepare<'a>( std::fs::create_dir(&tmp).map_err(|error| { EngineError::Restore(format!("rewind: stage {}: {error}", tmp.display())) })?; - write_journal( + write_staging_marker( &staging_journal_path, - &LegacyJournal { - target_generation: hex::encode(target.digest().as_bytes()), - repo_root: repo.clone(), - tmp: tmp.clone(), - phase: LegacyPhase::Materializing, - carried: Vec::new(), + &StagingMarker { + temporary: tmp.clone(), }, )?; let generation = store.generation(target).await?; @@ -686,14 +687,9 @@ fn recover_staging(path: &Path) -> Result<()> { Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), Err(error) => return Err(error.into()), }; - let journal: LegacyJournal = serde_json::from_str(&text) + let marker: StagingMarker = serde_json::from_str(&text) .map_err(|error| EngineError::Restore(format!("rewind staging journal: {error}")))?; - if journal.phase != LegacyPhase::Materializing { - return Err(EngineError::Restore( - "rewind staging journal has an invalid phase".into(), - )); - } - remove_any(&journal.tmp)?; + remove_any(&marker.temporary)?; std::fs::remove_file(path)?; Ok(()) } @@ -823,30 +819,16 @@ fn path_exists(path: &Path) -> Result { } } -fn write_journal(path: &Path, journal: &LegacyJournal) -> Result<()> { - let text = serde_json::to_string(journal) - .map_err(|error| EngineError::Restore(format!("encode journal: {error}")))?; +fn write_staging_marker(path: &Path, marker: &StagingMarker) -> Result<()> { + let text = serde_json::to_string(marker) + .map_err(|error| EngineError::Restore(format!("encode staging marker: {error}")))?; let tmp = path.with_extension("tmp"); - // Durability before visibility: the journal only helps if it is on the - // platter before the phase it describes begins. - // - // One writable handle carries all of it. `sync_all` is a `FlushFileBuffers` - // on Windows, which needs write access -- flushing a handle from - // `File::open` fails there with "access is denied" -- and the handle has - // to be closed before the rename, because Windows will not rename a file - // anyone still holds open. - let write = || -> std::io::Result<()> { - use std::io::Write; - let mut file = std::fs::File::create(&tmp)?; - file.write_all(text.as_bytes())?; - file.sync_all()?; - drop(file); - durable_rename(&tmp, path, RenameMode::Replace) - }; - write().map_err(|error| { - let _ = std::fs::remove_file(&tmp); - EngineError::Restore(format!("rewind: journal {}: {error}", path.display())) - })?; + let mut file = std::fs::File::create(&tmp)?; + use std::io::Write; + file.write_all(text.as_bytes())?; + file.sync_all()?; + drop(file); + durable_rename(&tmp, path, RenameMode::Replace)?; Ok(()) } @@ -1072,10 +1054,13 @@ mod tests { std::fs::write(tmp.join("partial.txt"), b"half").expect("seed"); let journal_path = work.path().join("journal.json"); let staging_path = journal_path.with_extension("staging.json"); - write( + write_staging_marker( &staging_path, - &journal(&repo, &tmp, LegacyPhase::Materializing), - ); + &StagingMarker { + temporary: tmp.clone(), + }, + ) + .expect("write staging marker"); recover(&journal_path).expect("recover"); assert_eq!(std::fs::read(repo.join("keep.txt")).expect("read"), b"live"); From e8d138c5f2d553c6c34423025010bcc87730f76e Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 18:11:14 +0100 Subject: [PATCH 099/106] Cache TypeScript qualification dependencies --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e5a6b2..40bc42a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,8 +51,9 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git + ~/.bun/install/cache sdk/target-* - key: qualify-${{ matrix.os }}-${{ hashFiles('sdk/Cargo.lock', 'sdk/rust/**/*.rs', 'worktrees/graphcoder/plugin-sdk-lab/Cargo.lock', 'worktrees/graphcoder/plugin-sdk-lab/crates/**/*.rs') }} + key: qualify-${{ matrix.os }}-${{ hashFiles('sdk/Cargo.lock', 'sdk/bun.lock', 'sdk/rust/**/*.rs', 'sdk/typescript/**/*.ts', 'worktrees/graphcoder/plugin-sdk-lab/Cargo.lock', 'worktrees/graphcoder/plugin-sdk-lab/crates/**/*.rs') }} restore-keys: | qualify-${{ matrix.os }}- - name: Run the shared bounded gate From 56cec8a64136b0dfb23c8265183c30dcec53824f Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 18:35:46 +0100 Subject: [PATCH 100/106] Use SDK host path restore Signed-off-by: Var1377 --- crates/acyclic/src/merge.rs | 11 +- crates/acyclic/src/rewind.rs | 263 ++++------------------------------- 2 files changed, 35 insertions(+), 239 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index 6ea5a25..e65c871 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -903,9 +903,8 @@ where if !page.has_more { break; } - after = Some( - next.ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))?, - ); + after = + Some(next.ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))?); } Ok(children) } @@ -1184,10 +1183,10 @@ pub async fn materialize_paths( dir: &Path, paths: &[PathBuf], ) -> Result<()> { - let mut checkout = store.checkout_exact(generation).await?; + let generation = store.generation(generation).await?; for path in paths { - crate::rewind::materialize_path_into_checkout( - &mut checkout, + crate::rewind::materialize_path_from_generation( + &generation, dir, path, crate::rewind::PathReplace::LiveMount, diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 65f4a76..270e58e 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -3,19 +3,19 @@ //! beside the repository, and journal every phase so kill -9 leaves the repo fully-old or //! fully-new — never mixed. -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use acyclic_fs::{ - durable_rename, exchange_native_entries, materialize_checkout_host_path, - prepare_native_exchange_with_recovery, publish_native_exchange, recover_native_exchange, - IdempotencyKey, MaterializeError, MaterializeOptions, NativeExchangeJournal, RenameMode, + durable_rename, prepare_native_exchange_with_recovery, publish_native_exchange, + recover_native_exchange, HostPathReplacement, HostPathRestore, IdempotencyKey, + MaterializeOptions, NativeExchangeJournal, RenameMode, }; use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; use serde::{Deserialize, Serialize}; use crate::exclude::Exclusions; -use crate::store::{LocalCheckout, Store}; +use crate::store::{LocalGeneration, Store}; use crate::{EngineError, Result}; const MAXIMUM_DIRECTORY_ENTRIES: u32 = 1_024; @@ -81,8 +81,8 @@ pub async fn restore_path( relative: &Path, ) -> Result { let root = store.repo_root.clone(); - let mut checkout = store.checkout_exact(target).await?; - materialize_path_into_checkout(&mut checkout, &root, relative, PathReplace::Atomic).await + let generation = store.generation(target).await?; + materialize_path_from_generation(&generation, &root, relative, PathReplace::Atomic).await } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -91,168 +91,40 @@ pub(crate) enum PathReplace { LiveMount, } -pub(crate) async fn materialize_path_into_checkout( - checkout: &mut LocalCheckout, +pub(crate) async fn materialize_path_from_generation( + generation: &LocalGeneration, root: &Path, relative: &Path, replace: PathReplace, ) -> Result { - validate_relative(relative)?; - let normalized = normalized_relative(relative); - let destination = root.join(relative); - ensure_real_parents(root, relative)?; - let parent = destination - .parent() - .ok_or_else(|| EngineError::Restore("path has no parent".into()))?; let cancel = CancellationToken::new(); - // The SDK requires an empty root and recreates the selected path below - // it. Keep ordinary restore staging outside the watched repository; - // live mounts must stage on the mount to permit the final rename. - let stage_parent = match replace { - PathReplace::Atomic => root - .parent() - .ok_or_else(|| EngineError::Restore("repository root has no parent".into()))?, - PathReplace::LiveMount => parent, - }; - let stage_root = create_restore_stage(stage_parent)?; - let staged = stage_root.join(&normalized); - let materialized = materialize_checkout_host_path( - checkout, - &normalized, - &MaterializeOptions { - destination: stage_root.clone(), - maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, - maximum_extent_spans: MAXIMUM_EXTENT_SPANS, - transfer_bytes: TRANSFER_BYTES, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await; - if matches!( - materialized.as_ref().map_err(|failure| &failure.error), - Err(MaterializeError::MissingPath) - ) { - std::fs::remove_dir_all(&stage_root)?; - // Faithful restore of an absent path: remove it if it exists now. - return match std::fs::symlink_metadata(&destination) { - Ok(metadata) => { - if metadata.is_dir() { - std::fs::remove_dir_all(&destination)?; - } else { - std::fs::remove_file(&destination)?; - } - Ok(RestoreOutcome { - path: relative.to_path_buf(), - action: RestoreAction::Removed, - }) - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => match replace { - PathReplace::Atomic => Err(EngineError::Restore(format!( - "{} does not exist at that checkpoint or in the tree", - relative.display() - ))), - PathReplace::LiveMount => Ok(RestoreOutcome { - path: relative.to_path_buf(), - action: RestoreAction::Removed, - }), + let restored = generation + .restore_host_path( + relative, + match replace { + PathReplace::Atomic => HostPathReplacement::Atomic, + PathReplace::LiveMount => HostPathReplacement::LiveMount, }, - Err(error) => Err(error.into()), - }; - } - if let Err(error) = materialized { - let _ = std::fs::remove_dir_all(&stage_root); - return Err(EngineError::Restore(format!("materialize path: {error}"))); - } - ensure_real_parents(root, relative)?; - - // A live mount must observe ordinary remove/rename operations through - // its driver. A user restore exchanges an existing node atomically. - match std::fs::symlink_metadata(&destination) { - Ok(_) => match replace { - PathReplace::Atomic => { - // An exchange may have published the new node before a - // durability error. Keep both staged and scratch trees. - exchange_native_entries(&destination, &staged) - .map_err(|error| EngineError::Restore(format!("exchange path: {error}")))?; - } - PathReplace::LiveMount => replace_live_mount(&staged, &destination, parent)?, - }, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - let renamed = match replace { - PathReplace::Atomic => durable_rename(&staged, &destination, RenameMode::NoReplace), - PathReplace::LiveMount => std::fs::rename(&staged, &destination), - }; - if let Err(error) = renamed { - return Err(error.into()); - } - } - Err(error) => { - return Err(error.into()); - } - } - std::fs::remove_dir_all(&stage_root)?; - #[cfg(unix)] - if replace == PathReplace::Atomic { - sync_parent(&stage_root)?; - } + &MaterializeOptions { + destination: root.to_path_buf(), + maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, + maximum_extent_spans: MAXIMUM_EXTENT_SPANS, + transfer_bytes: TRANSFER_BYTES, + }, + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(|error| EngineError::Restore(error.to_string()))?; Ok(RestoreOutcome { path: relative.to_path_buf(), - action: RestoreAction::Restored, + action: match restored.value { + HostPathRestore::Restored => RestoreAction::Restored, + HostPathRestore::Removed => RestoreAction::Removed, + }, }) } -fn replace_live_mount(staged: &Path, destination: &Path, parent: &Path) -> Result<()> { - let backup_root = create_restore_stage(parent)?; - let backup = backup_root.join("old"); - if let Err(error) = std::fs::rename(destination, &backup) { - let _ = std::fs::remove_dir(&backup_root); - return Err(error.into()); - } - if let Err(error) = std::fs::rename(staged, destination) { - if let Err(rollback) = std::fs::rename(&backup, destination) { - return Err(EngineError::Restore(format!( - "materialize rename failed: {error}; old node remains at {} after rollback failed: {rollback}", - backup.display() - ))); - } - let _ = std::fs::remove_dir(&backup_root); - return Err(error.into()); - } - remove_any(&backup)?; - std::fs::remove_dir(&backup_root)?; - Ok(()) -} - -fn normalized_relative(relative: &Path) -> PathBuf { - relative - .components() - .filter_map(|component| match component { - Component::Normal(name) => Some(name), - _ => None, - }) - .collect() -} - -fn create_restore_stage(parent: &Path) -> Result { - for _ in 0..16 { - let sequence = PARK_SEQUENCE.fetch_add(1, Ordering::Relaxed); - let stage = parent.join(format!( - ".{}-restore-{}-{sequence}", - crate::product::NAME, - std::process::id() - )); - match std::fs::create_dir(&stage) { - Ok(()) => return Ok(stage), - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(error) => return Err(error.into()), - } - } - Err(EngineError::Restore( - "could not allocate a unique restore stage".into(), - )) -} - pub(crate) fn validate_relative(relative: &Path) -> Result>> { let config = crate::store::volume_config(); let namespace = acyclic_fs::host_path_to_namespace(relative, config.profile, config.limits) @@ -269,48 +141,6 @@ pub(crate) fn validate_relative(relative: &Path) -> Result>> { .collect()) } -/// Create missing ancestors one component at a time, refusing symlinks and -/// Windows reparse points before a path restore touches its destination. -fn ensure_real_parents(root: &Path, relative: &Path) -> Result<()> { - fn check_real_directory(path: &Path) -> Result<()> { - let metadata = std::fs::symlink_metadata(path)?; - #[cfg(windows)] - let reparse = { - use std::os::windows::fs::MetadataExt; - metadata.file_attributes() & 0x400 != 0 // FILE_ATTRIBUTE_REPARSE_POINT - }; - #[cfg(not(windows))] - let reparse = false; - if !metadata.is_dir() || metadata.file_type().is_symlink() || reparse { - return Err(EngineError::Restore(format!( - "restore parent {} is not a real directory", - path.display() - ))); - } - Ok(()) - } - - check_real_directory(root)?; - let mut cursor = root.to_path_buf(); - if let Some(parent) = relative.parent() { - for component in parent.components() { - let Component::Normal(name) = component else { - continue; - }; - cursor.push(name); - match std::fs::symlink_metadata(&cursor) { - Ok(_) => check_real_directory(&cursor)?, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - std::fs::create_dir(&cursor)?; - check_real_directory(&cursor)?; - } - Err(error) => return Err(error.into()), - } - } - } - Ok(()) -} - pub(crate) fn remove_any(path: &Path) -> std::io::Result<()> { match std::fs::symlink_metadata(path) { Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path), @@ -934,39 +764,6 @@ mod tests { Ok(()) } - #[cfg(unix)] - #[test] - fn path_restore_rejects_symlinked_parent() -> Result<()> { - let work = tempfile::tempdir()?; - let root = work.path().join("repo"); - let outside = work.path().join("outside"); - std::fs::create_dir(&root)?; - std::fs::create_dir(&outside)?; - std::os::unix::fs::symlink(&outside, root.join("dir"))?; - assert!(ensure_real_parents(&root, Path::new("dir/file")).is_err()); - assert!(!outside.join("file").exists()); - Ok(()) - } - - #[cfg(windows)] - #[test] - fn path_restore_rejects_reparse_parent_when_symlinks_are_available() -> Result<()> { - let work = tempfile::tempdir()?; - let root = work.path().join("repo"); - let outside = work.path().join("outside"); - std::fs::create_dir(&root)?; - std::fs::create_dir(&outside)?; - if let Err(error) = std::os::windows::fs::symlink_dir(&outside, root.join("dir")) { - if error.kind() == std::io::ErrorKind::PermissionDenied { - return Ok(()); - } - return Err(error.into()); - } - assert!(ensure_real_parents(&root, Path::new("dir/file")).is_err()); - assert!(!outside.join("file").exists()); - Ok(()) - } - fn recover(journal_path: &Path) -> Result> { super::recover(journal_path) } From 0daffc83c76822e0a2b11eab3f35f88411d1b100 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 18:38:35 +0100 Subject: [PATCH 101/106] Use canonical SDK materialization bounds Signed-off-by: Var1377 --- crates/acyclic/src/rewind.rs | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index 270e58e..f515513 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -18,9 +18,6 @@ use crate::exclude::Exclusions; use crate::store::{LocalGeneration, Store}; use crate::{EngineError, Result}; -const MAXIMUM_DIRECTORY_ENTRIES: u32 = 1_024; -const MAXIMUM_EXTENT_SPANS: u32 = 65_536; -const TRANSFER_BYTES: u64 = 8 * 1024 * 1024; static PARK_SEQUENCE: AtomicU64 = AtomicU64::new(0); pub(crate) fn publication_key(generation: GenerationId) -> Result { @@ -105,12 +102,7 @@ pub(crate) async fn materialize_path_from_generation( PathReplace::Atomic => HostPathReplacement::Atomic, PathReplace::LiveMount => HostPathReplacement::LiveMount, }, - &MaterializeOptions { - destination: root.to_path_buf(), - maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, - maximum_extent_spans: MAXIMUM_EXTENT_SPANS, - transfer_bytes: TRANSFER_BYTES, - }, + &MaterializeOptions::native(root), WorkCounters::UNBOUNDED, &cancel, ) @@ -389,12 +381,7 @@ pub(crate) async fn prepare<'a>( let cancel = CancellationToken::new(); generation .materialize( - &MaterializeOptions { - destination: tmp.clone(), - maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, - maximum_extent_spans: MAXIMUM_EXTENT_SPANS, - transfer_bytes: TRANSFER_BYTES, - }, + &MaterializeOptions::native(&tmp), WorkCounters::UNBOUNDED, &cancel, ) From 035ac0e5eb8734305b3b1145b34d417a666fc6f7 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 18:47:38 +0100 Subject: [PATCH 102/106] Batch SDK host path restores Signed-off-by: Var1377 --- crates/acyclic/src/merge.rs | 16 ++++++++-------- crates/acyclic/src/rewind.rs | 24 +++--------------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/crates/acyclic/src/merge.rs b/crates/acyclic/src/merge.rs index e65c871..e1a4a50 100644 --- a/crates/acyclic/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -1184,15 +1184,15 @@ pub async fn materialize_paths( paths: &[PathBuf], ) -> Result<()> { let generation = store.generation(generation).await?; - for path in paths { - crate::rewind::materialize_path_from_generation( - &generation, - dir, - path, - crate::rewind::PathReplace::LiveMount, + generation + .restore_host_paths( + paths, + acyclic_fs::HostPathReplacement::LiveMount, + &acyclic_fs::MaterializeOptions::native(dir), + &acyclic_fs::CancellationToken::new(), ) - .await?; - } + .await + .map_err(EngineError::fs("materialize paths"))?; Ok(()) } diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index f515513..bdb7e5e 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -15,7 +15,7 @@ use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; use serde::{Deserialize, Serialize}; use crate::exclude::Exclusions; -use crate::store::{LocalGeneration, Store}; +use crate::store::Store; use crate::{EngineError, Result}; static PARK_SEQUENCE: AtomicU64 = AtomicU64::new(0); @@ -79,30 +79,12 @@ pub async fn restore_path( ) -> Result { let root = store.repo_root.clone(); let generation = store.generation(target).await?; - materialize_path_from_generation(&generation, &root, relative, PathReplace::Atomic).await -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum PathReplace { - Atomic, - LiveMount, -} - -pub(crate) async fn materialize_path_from_generation( - generation: &LocalGeneration, - root: &Path, - relative: &Path, - replace: PathReplace, -) -> Result { let cancel = CancellationToken::new(); let restored = generation .restore_host_path( relative, - match replace { - PathReplace::Atomic => HostPathReplacement::Atomic, - PathReplace::LiveMount => HostPathReplacement::LiveMount, - }, - &MaterializeOptions::native(root), + HostPathReplacement::Atomic, + &MaterializeOptions::native(&root), WorkCounters::UNBOUNDED, &cancel, ) From e82de7fd88c5148477ab13daf11a42572594b530 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 19:19:03 +0100 Subject: [PATCH 103/106] Reconcile crash recovery before serving Signed-off-by: Var1377 --- Cargo.lock | 13 +++++++++++ crates/acyclic/src/main.rs | 32 ++++++++++++++++++++++---- crates/acyclic/src/pipeline.rs | 1 + crates/acyclic/src/proto.rs | 1 - crates/acyclic/src/rewind.rs | 26 +++++++++++++++++---- crates/acyclic/src/server.rs | 24 ++++++++++--------- crates/acyclic/tests/restart_rewind.rs | 1 + 7 files changed, 76 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ecdbe4d..da452c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,6 +34,7 @@ dependencies = [ "async-trait", "blake3", "bytes", + "cap-fs-ext", "cap-primitives", "cap-std", "cc", @@ -335,6 +336,18 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cap-fs-ext" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ff379b70af8e08307a8f65e7040c7301cb4a572538ade16b4984f0da77847f" +dependencies = [ + "cap-primitives", + "cap-std", + "io-lifetimes 3.0.1", + "windows-sys 0.61.2", +] + [[package]] name = "cap-primitives" version = "4.0.3" diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index 3335e41..739676e 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -240,14 +240,37 @@ fn main() { std::process::exit(run(cli, Path::new("."))); } let repo_arg = cli.repo.clone().unwrap_or_else(|| PathBuf::from(".")); - let repo_arg = acyclic::rewind::recover_before_repo_open(&repo_arg).unwrap_or_else(|error| { - eprintln!("{}: rewind recovery: {error}", product::NAME); - std::process::exit(1); - }); + let (repo_arg, recovered) = acyclic::rewind::recover_before_repo_open(&repo_arg) + .unwrap_or_else(|error| { + eprintln!("{}: rewind recovery: {error}", product::NAME); + std::process::exit(1); + }); let repo = repo_arg.canonicalize().unwrap_or_else(|error| { eprintln!("{}: bad repo path: {error}", product::NAME); std::process::exit(1); }); + let recovery = match recovered { + Some(recovered) if recovered.reconcile_head => (|| -> Result<(), String> { + let config = acyclic::config::Config::load(&repo).map_err(|error| error.to_string())?; + let stores_root = config.store_dir.as_ref().map(PathBuf::from); + let paths = acyclic::store::StorePaths::for_repo(&repo, stores_root.as_deref()) + .map_err(|error| error.to_string())?; + let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; + let mut store = runtime + .block_on(acyclic::store::Store::open(&repo, paths)) + .map_err(|error| error.to_string())?; + runtime + .block_on(acyclic::rewind::recover_workspace(&mut store, recovered)) + .map_err(|error| error.to_string())?; + acyclic::rewind::finish_recovery(&repo).map_err(|error| error.to_string()) + })(), + Some(_) => acyclic::rewind::finish_recovery(&repo).map_err(|error| error.to_string()), + None => Ok(()), + }; + if let Err(error) = recovery { + eprintln!("{}: finish rewind recovery: {error}", product::NAME); + std::process::exit(1); + } if cli.repo.is_none() && stranded_in_trash(&repo) { // A rewind or promote swaps the repo directory's inode; a shell that // was inside it now resolves its cwd to the replaced tree in trash. @@ -964,7 +987,6 @@ fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), Str .map_or_else(|| "none".into(), |id| format!("#{id}")) ); println!("unpublished: {}", info.unpublished); - println!("store size: {}", human_bytes(info.store_bytes)); if info.mount_available { println!("mounts: {} (forks mount)", info.mount_provider); } else { diff --git a/crates/acyclic/src/pipeline.rs b/crates/acyclic/src/pipeline.rs index 97866e4..7ebffa8 100644 --- a/crates/acyclic/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -708,6 +708,7 @@ pub fn spawn( let (sender, receiver) = mpsc::channel(1024); let thread = std::thread::Builder::new() .name(format!("{}-pipeline", crate::product::NAME)) + .stack_size(8 * 1024 * 1024) .spawn(move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_time() diff --git a/crates/acyclic/src/proto.rs b/crates/acyclic/src/proto.rs index 8375ce8..58c6e07 100644 --- a/crates/acyclic/src/proto.rs +++ b/crates/acyclic/src/proto.rs @@ -389,7 +389,6 @@ pub struct StatusInfo { pub state: String, pub last_checkpoint: Option, pub unpublished: u64, - pub store_bytes: u64, pub repo_root: String, /// Mount provider this daemon would use for forks. #[serde(default)] diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs index bdb7e5e..c123578 100644 --- a/crates/acyclic/src/rewind.rs +++ b/crates/acyclic/src/rewind.rs @@ -189,6 +189,8 @@ pub struct RecoveredSwap { pub old_tree: Option, /// Generation named by the durable journal. pub target: GenerationId, + /// The durable SDK head was part of this operation and must be reconciled. + pub reconcile_head: bool, } pub async fn recover_workspace(store: &mut Store, recovered: RecoveredSwap) -> Result<()> { @@ -258,7 +260,7 @@ fn locator_path(repo_root: &Path) -> Result { /// Recovers before loading the repository's config or canonicalizing its root. /// Both operations can fail after the first Windows rename has removed it. -pub fn recover_before_repo_open(repo_root: &Path) -> Result { +pub fn recover_before_repo_open(repo_root: &Path) -> Result<(PathBuf, Option)> { let canonical = match repo_root.canonicalize() { Ok(path) => path, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { @@ -302,7 +304,7 @@ pub fn recover_before_repo_open(repo_root: &Path) -> Result { let locator_path = locator_path(&canonical)?; let text = match std::fs::read_to_string(&locator_path) { Ok(text) => text, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(canonical), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok((canonical, None)), Err(error) => return Err(error.into()), }; let locator: RecoveryLocator = serde_json::from_str(&text) @@ -310,9 +312,18 @@ pub fn recover_before_repo_open(repo_root: &Path) -> Result { if locator.repo_root != canonical { return Err(EngineError::Restore("rewind locator repo mismatch".into())); } - recover(&locator.journal)?; - std::fs::remove_file(locator_path)?; - Ok(canonical) + let recovered = recover(&locator.journal)?; + Ok((canonical, recovered)) +} + +/// Acknowledges that the store head was reconciled after pre-open recovery. +pub fn finish_recovery(repo_root: &Path) -> Result<()> { + let path = locator_path(repo_root)?; + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } } /// Executes a rewind against the store's repo. Called from the pipeline with @@ -475,6 +486,7 @@ pub fn recover(journal_path: &Path) -> Result> { published: outcome.published, old_tree: outcome.displaced, target, + reconcile_head: true, })); } recover_legacy(journal_path, &text) @@ -512,6 +524,7 @@ fn recover_legacy(journal_path: &Path, text: &str) -> Result { @@ -598,6 +611,7 @@ fn recover_legacy(journal_path: &Path, text: &str) -> Result Result<(), String> { ); phase = std::time::Instant::now(); }; - let repo_root = + let (repo_root, early_recovered) = rewind::recover_before_repo_open(repo_root).map_err(|error| error.to_string())?; lap("rewind recovery before repo open"); let config = Config::load(&repo_root).map_err(|error| error.to_string())?; @@ -165,7 +165,10 @@ pub fn run(repo_root: &Path) -> Result<(), String> { // Finish or unwind any rewind that a crash interrupted before serving // stateful operations. Fork cleanup is delayed until forks are requested. - let recovered = rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?; + let recovered = match early_recovered { + Some(recovered) => Some(recovered), + None => rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?, + }; lap("rewind journal recovery"); let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; @@ -183,6 +186,7 @@ pub fn run(repo_root: &Path) -> Result<(), String> { runtime .block_on(rewind::recover_workspace(&mut store, recovered)) .map_err(|error| error.to_string())?; + rewind::finish_recovery(&repo_root).map_err(|error| error.to_string())?; } let repo_root = store.repo_root.clone(); lap("store open"); @@ -349,7 +353,6 @@ impl Server { state: format!("{:?}", status.state).to_lowercase(), last_checkpoint: status.last_checkpoint, unpublished: status.unpublished, - store_bytes: 0, repo_root: self.repo_root.display().to_string(), mount_provider: self.mounts.provider.to_owned(), mount_available: self.mounts.available, @@ -1409,6 +1412,12 @@ impl Server { let mut mount = self.fork_mount.lock().await; let route = route_name(id)?; let last_route = mount.router.route_count() == 1; + if !last_route { + if let Some(session) = mount.session.as_ref() { + tokio::task::block_in_place(|| session.invalidate(&route)) + .map_err(|error| format!("invalidate {id}: {error:?}"))?; + } + } if last_route { if let Some(session) = mount.session.as_mut() { tokio::task::block_in_place(|| session.stop()) @@ -1423,16 +1432,9 @@ impl Server { if !tokio::task::block_in_place(|| mount.router.remove_route(&route)) { return Err(format!("fork {id} has no mount route")); } - // The kernel may hold a positive entry cache for the removed name - // (FSKit caches until told otherwise): invalidate it eagerly. - if let Some(session) = mount.session.as_ref() { - tokio::task::block_in_place(|| session.invalidate(&route)) - .map_err(|error| format!("invalidate {id}: {error:?}"))?; - } if last_route { if let Some(root) = fork::forks_mount_root(&self.repo_root) { - std::fs::remove_dir_all(root) - .map_err(|error| format!("remove fork mount root: {error}"))?; + let _ = std::fs::remove_dir(root); } } Ok(()) diff --git a/crates/acyclic/tests/restart_rewind.rs b/crates/acyclic/tests/restart_rewind.rs index 645df48..d1f9686 100644 --- a/crates/acyclic/tests/restart_rewind.rs +++ b/crates/acyclic/tests/restart_rewind.rs @@ -41,6 +41,7 @@ fn cli_recovers_a_missing_repo_before_loading_config() -> Result<(), Box Date: Sat, 19 Sep 2026 19:19:10 +0100 Subject: [PATCH 104/106] Guard SDK head recovery by journal kind Signed-off-by: Var1377 --- crates/acyclic/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs index 8756b4e..01dab72 100644 --- a/crates/acyclic/src/store.rs +++ b/crates/acyclic/src/store.rs @@ -253,7 +253,7 @@ impl Store { &mut self, recovered: &crate::rewind::RecoveredSwap, ) -> Result<()> { - if !recovered.published { + if !recovered.reconcile_head || !recovered.published { return Ok(()); } let current = self From b3d6d5fdbb85f351349cdd9b740fb3a3534d71a2 Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 19:20:53 +0100 Subject: [PATCH 105/106] Remove fork copy compatibility surface Signed-off-by: Var1377 --- crates/acyclic/src/install.rs | 4 +--- crates/acyclic/src/main.rs | 7 +++---- crates/acyclic/src/proto.rs | 7 ------- crates/acyclic/src/server.rs | 2 -- docs/windows-verification.md | 9 ++++----- 5 files changed, 8 insertions(+), 21 deletions(-) diff --git a/crates/acyclic/src/install.rs b/crates/acyclic/src/install.rs index 1698d07..fcf19b7 100644 --- a/crates/acyclic/src/install.rs +++ b/crates/acyclic/src/install.rs @@ -1194,9 +1194,7 @@ effective values in ONE line before the first fork, e.g. Depth starts at 1 and increases by one per round. 1. `{{name}} fork -n `. It records the fork base as a checkpoint - itself, so do not checkpoint first. Note each id and path and whether - it says `(mount)` or `(copy)`. Copy forks cost time proportional to - the tree: keep them few and short-lived. + itself, so do not checkpoint first. Note each id and mounted path. 2. Dispatch ALL subagents in one turn, one per fork, using the CHILD PROMPT below. Do not keep one approach for yourself. 3. **Freeze.** Make NO edits to the real tree while forks are live. A diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index 739676e..fc3a783 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -891,7 +891,7 @@ fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), Str return Err("unexpected reply".into()); }; for entry in &entries { - println!("fork {} ({}) {}", entry.id, entry.mode, entry.path); + println!("fork {} {}", entry.id, entry.path); } println!( "{} fork(s) ready — work in them freely; `{NAME} promote ` keeps a winner", @@ -918,9 +918,8 @@ fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), Str ) }; println!( - "{} {} {} {} base {}{conflict}", + "{} {} {} base {}{conflict}", entry.id, - entry.mode, age(entry.created_at), entry.path, short_hex(&entry.base) @@ -991,7 +990,7 @@ fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), Str println!("mounts: {} (forks mount)", info.mount_provider); } else { println!( - "mounts: unavailable ({}) — forks copy", + "mounts: unavailable ({}) — forks disabled", info.mount_reason.as_deref().unwrap_or("unknown reason") ); } diff --git a/crates/acyclic/src/proto.rs b/crates/acyclic/src/proto.rs index 58c6e07..5917f8d 100644 --- a/crates/acyclic/src/proto.rs +++ b/crates/acyclic/src/proto.rs @@ -317,9 +317,6 @@ pub struct SummaryInfo { pub struct ForkEntry { pub id: String, pub path: String, - /// "mount" (routed native mount) or "copy" (materialized directory). - #[serde(default = "default_fork_mode")] - pub mode: String, /// Hex of the published generation the fork was cut from. pub base: String, pub created_at: i64, @@ -587,7 +584,3 @@ pub struct DiffEntry { #[serde(default)] pub ignored: bool, } - -fn default_fork_mode() -> String { - "mount".to_owned() -} diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index 4b43f48..4dbf0e6 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -733,7 +733,6 @@ impl Server { let entry = proto::ForkEntry { id: id.clone(), path: root.join(&id).display().to_string(), - mode: "mount".to_owned(), base: acyclic::generation_hex(seed.base), created_at: unix_now(), session_id: session_id.clone(), @@ -1932,7 +1931,6 @@ fn clone_entry(entry: &proto::ForkEntry) -> proto::ForkEntry { proto::ForkEntry { id: entry.id.clone(), path: entry.path.clone(), - mode: entry.mode.clone(), base: entry.base.clone(), created_at: entry.created_at, session_id: entry.session_id.clone(), diff --git a/docs/windows-verification.md b/docs/windows-verification.md index d9967e4..c96252e 100644 --- a/docs/windows-verification.md +++ b/docs/windows-verification.md @@ -168,15 +168,14 @@ correct. ## Known limits -- **Forks are always copies.** `ProjFS` mounts and +- **Forks require a writable native mount.** `ProjFS` mounts and projects a fork correctly — the tree appears and reads back fine — but writes into the projection stop at the `ProjFS` local cache and never reach the overlay checkout. A mounted fork therefore looked like it worked while `fork-diff` reported no changes and `promote` landed nothing: it silently - ate the work. `mount_capability()` now reports mounts unavailable on - Windows, so forks take the copy path, which is verified end to end (write, - `fork-diff`, `promote` all behave). Revisit if the sdk's `ProjFS` provider - gains write-back. + ate the work. `mount_capability()` reports mounts unavailable on Windows + until the SDK's `ProjFS` provider gains write-back, and fork creation fails + explicitly instead of materializing a full-tree compatibility copy. - **The tree exchange is not atomic.** `RENAME_EXCHANGE` has no Windows equivalent, so `atomic_exchange` does three renames through a scratch From dd0d03429b8df73a505f5d8ad0d3b18c7e00c34e Mon Sep 17 00:00:00 2001 From: Var1377 Date: Sat, 19 Sep 2026 19:21:06 +0100 Subject: [PATCH 106/106] Trigger cross-platform qualification Signed-off-by: Var1377